boost/website-v2
2
0
Fork 0
mirror of https://github.com/boostorg/website-v2.git synced 2026-02-27 17:42:08 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'main' of github.com:revsys/boost.org

Browse Source
This commit is contained in:
Lacey Williams Henschel
2023-01-04 13:54:05 -08:00
parent 79120e12cb e12fed1dfb
commit 4e2ede1603
10 changed files with 319 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config/test_settings.py
View File

@@ -25,3 +25,5 @@ MIGRATION_MODULES = DisableMigrations()
# User a faster password hasher
PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
GITHUB_TOKEN = "changeme"

57
libraries/github.py
View File

@@ -1,12 +1,11 @@
import base64
import os
import re
import requests
import structlog
from dateutil.parser import ParserError, parse
from ghapi.all import GhApi, paged
from fastcore.xtras import obj2dict
from ghapi.all import GhApi, paged
from .models import Category, Issue, Library, PullRequest
from .utils import parse_date
@@ -37,6 +36,9 @@ def repo_issues(owner, repo, state="all", issues_only=True):
Note: The GitHub API considers both PRs and Issues to be "Issues" and does not
support filtering in the request, so to exclude PRs from the list of issues, we
do some manual filtering of the results
Note: GhApi() returns results as AttrDict objects:
https://fastcore.fast.ai/basics.html#attrdict
"""
api = get_api()
pages = list(
@@ -64,7 +66,12 @@ def repo_issues(owner, repo, state="all", issues_only=True):
def repo_prs(owner, repo, state="all"):
"""Get all PRs for a repo"""
"""
Get all PRs for a repo
Note: GhApi() returns results as AttrDict objects:
https://fastcore.fast.ai/basics.html#attrdict
"""
api = get_api()
pages = list(
paged(
@@ -192,11 +199,7 @@ class LibraryUpdater:
meta = self.get_library_metadata(repo=name)
github_data = self.get_library_github_data(owner=self.owner, repo=name)
try:
last_github_update = parse(github_data.get("updated_at"))
except ParserError:
last_github_update = None
last_github_update = parse_date(github_data.get("updated_at", ""))
github_url = f"https://github.com/boostorg/{name}/"
if type(meta) is list:
@@ -262,7 +265,9 @@ class LibraryUpdater:
self.logger.info("update_all_libraries_metadata", library_count=len(libs))
for lib in libs:
self.update_library(lib)
library = self.update_library(lib)
github_updater = GithubUpdater(owner=self.owner, library=library)
github_updater.update()
def update_categories(self, obj, categories):
"""Update all of the categories for an object"""
@@ -291,6 +296,8 @@ class LibraryUpdater:
logger.info("library_udpated")
return obj
except Exception:
logger.exception("library_update_failed")
@@ -302,13 +309,12 @@ class GithubUpdater:
for the site
"""
def __init__(self, owner, repo):
def __init__(self, owner="boostorg", library=None):
self.owner = owner
self.repo = repo
self.logger = logger.bind(owner=owner, repo=repo)
self.library = library
self.logger = logger.bind(owner=owner, library=library)
def update(self):
# FIXME: Write a test
self.logger.info("update_github_repo")
try:
@@ -322,6 +328,7 @@ class GithubUpdater:
self.logger.exception("update_prs_error")
def update_issues(self):
"""Update all issues for a library"""
self.logger.info("updating_repo_issues")
issues_data = repo_issues(
@@ -365,11 +372,10 @@ class GithubUpdater:
issue_github_id=issue_dict.get("id"),
exc_msg=str(e),
)
continue
logger.info(
"issue_updated_successfully",
issue_id=issue.id,
created=created,
created_issue=created,
issue_github_id=issue.github_id,
)
@@ -377,9 +383,10 @@ class GithubUpdater:
"""Update all PRs for a library"""
self.logger.info("updating_repo_prs")
prs_data = repo_prs(self.owner, self.library.name)
prs_data = repo_prs(self.owner, self.library.name, state="all")
for pr_dict in prs_data:
# Get the date information
closed_at = None
merged_at = None
@@ -395,10 +402,8 @@ class GithubUpdater:
if pr_dict.get("created_at"):
created_at = parse_date(pr_dict["created_at"])
if pr_dict.get("modified_at"):
modified_at = parse_date(pr_dict["modified_at"])
breakpoint()
if pr_dict.get("updated_at"):
modified_at = parse_date(pr_dict["updated_at"])
try:
pull_request, created = PullRequest.objects.update_or_create(
@@ -421,11 +426,9 @@ class GithubUpdater:
pr_github_id=pr_dict.get("id"),
exc_msg=str(e),
)
continue
logger.info(
"pr_updated_successfully",
pull_request_id=pull_request.id,
created=created,
pull_request_github_id=pull_request.github_id,
"pull_request_updated_successfully",
pr_id=pull_request.id,
created_pr=created,
pr_github_id=pull_request.github_id,
)

10
libraries/management/commands/update_libraries.py
View File

@@ -5,5 +5,11 @@ from libraries.github import LibraryUpdater
@click.command()
def command():
l = LibraryUpdater()
l.update_libraries()
"""
Calls the LibraryUpdater, which retrieves the active boost libraries
from the Boost repo and updates the models in our database with the latest
information on that library (repo) and its issues, pull requests, and related
objects from GitHub.
"""
updater = LibraryUpdater()
updater.update_libraries()

363
libraries/tests/fixtures.py
View File

@@ -1,4 +1,5 @@
import pytest
from fastcore.xtras import dict2obj
from model_bakery import baker
@@ -103,330 +104,44 @@ def github_api_repo_issues_response(db):
@pytest.fixture
def github_api_repo_prs_response(db):
"""Returns the response from GhApi().pulls.list, already paged"""
return {
"url": "https://api.github.com/repos/boostorg/system/pulls/90",
"id": 1032140532,
"node_id": "PR_kwDOAHPQi849hTb0",
"html_url": "https://github.com/boostorg/system/pull/90",
"diff_url": "https://github.com/boostorg/system/pull/90.diff",
"patch_url": "https://github.com/boostorg/system/pull/90.patch",
"issue_url": "https://api.github.com/repos/boostorg/system/issues/90",
"number": 90,
"state": "open",
"locked": False,
"title": "add boost_system.natvis and interface source files",
"user": {
"login": "vinniefalco",
"id": 1503976,
"node_id": "MDQ6VXNlcjE1MDM5NzY=",
"avatar_url": "https://avatars.githubusercontent.com/u/1503976?v=4",
"gravatar_id": "",
"url": "https://api.github.com/users/vinniefalco",
"html_url": "https://github.com/vinniefalco",
"followers_url": "https://api.github.com/users/vinniefalco/followers",
"following_url": "https://api.github.com/users/vinniefalco/following{/other_user}",
"gists_url": "https://api.github.com/users/vinniefalco/gists{/gist_id}",
"starred_url": "https://api.github.com/users/vinniefalco/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/vinniefalco/subscriptions",
"organizations_url": "https://api.github.com/users/vinniefalco/orgs",
"repos_url": "https://api.github.com/users/vinniefalco/repos",
"events_url": "https://api.github.com/users/vinniefalco/events{/privacy}",
"received_events_url": "https://api.github.com/users/vinniefalco/received_events",
"type": "User",
"site_admin": False,
},
"body": None,
"created_at": "2022-08-21T22:24:43Z",
"updated_at": "2022-08-21T22:24:43Z",
"closed_at": None,
"merged_at": None,
"merge_commit_sha": "37653ac206475a046d7e7abadaf823430e564572",
"assignee": None,
"assignees": [],
"requested_reviewers": [],
"requested_teams": [],
"labels": [],
"milestone": None,
"draft": False,
"commits_url": "https://api.github.com/repos/boostorg/system/pulls/90/commits",
"review_comments_url": "https://api.github.com/repos/boostorg/system/pulls/90/comments",
"review_comment_url": "https://api.github.com/repos/boostorg/system/pulls/comments{/number}",
"comments_url": "https://api.github.com/repos/boostorg/system/issues/90/comments",
"statuses_url": "https://api.github.com/repos/boostorg/system/statuses/fe48c3058daaa31da6c50c316d63aa5f185dacb8",
"head": {
"label": "vinniefalco:natvis",
"ref": "natvis",
"sha": "fe48c3058daaa31da6c50c316d63aa5f185dacb8",
"user": {
"login": "vinniefalco",
"id": 1503976,
"node_id": "MDQ6VXNlcjE1MDM5NzY=",
"avatar_url": "https://avatars.githubusercontent.com/u/1503976?v=4",
"gravatar_id": "",
"url": "https://api.github.com/users/vinniefalco",
"html_url": "https://github.com/vinniefalco",
"followers_url": "https://api.github.com/users/vinniefalco/followers",
"following_url": "https://api.github.com/users/vinniefalco/following{/other_user}",
"gists_url": "https://api.github.com/users/vinniefalco/gists{/gist_id}",
"starred_url": "https://api.github.com/users/vinniefalco/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/vinniefalco/subscriptions",
"organizations_url": "https://api.github.com/users/vinniefalco/orgs",
"repos_url": "https://api.github.com/users/vinniefalco/repos",
"events_url": "https://api.github.com/users/vinniefalco/events{/privacy}",
"received_events_url": "https://api.github.com/users/vinniefalco/received_events",
"type": "User",
"site_admin": False,
},
"repo": {
"id": 526406204,
"node_id": "R_kgDOH2BSPA",
"name": "boost-system",
"full_name": "vinniefalco/boost-system",
"private": False,
"owner": {
"login": "vinniefalco",
"id": 1503976,
"node_id": "MDQ6VXNlcjE1MDM5NzY=",
"avatar_url": "https://avatars.githubusercontent.com/u/1503976?v=4",
"gravatar_id": "",
"url": "https://api.github.com/users/vinniefalco",
"html_url": "https://github.com/vinniefalco",
"followers_url": "https://api.github.com/users/vinniefalco/followers",
"following_url": "https://api.github.com/users/vinniefalco/following{/other_user}",
"gists_url": "https://api.github.com/users/vinniefalco/gists{/gist_id}",
"starred_url": "https://api.github.com/users/vinniefalco/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/vinniefalco/subscriptions",
"organizations_url": "https://api.github.com/users/vinniefalco/orgs",
"repos_url": "https://api.github.com/users/vinniefalco/repos",
"events_url": "https://api.github.com/users/vinniefalco/events{/privacy}",
"received_events_url": "https://api.github.com/users/vinniefalco/received_events",
"type": "User",
"site_admin": False,
},
"html_url": "https://github.com/vinniefalco/boost-system",
"description": "Boost.org system module ",
"fork": True,
"url": "https://api.github.com/repos/vinniefalco/boost-system",
"forks_url": "https://api.github.com/repos/vinniefalco/boost-system/forks",
"keys_url": "https://api.github.com/repos/vinniefalco/boost-system/keys{/key_id}",
"collaborators_url": "https://api.github.com/repos/vinniefalco/boost-system/collaborators{/collaborator}",
"teams_url": "https://api.github.com/repos/vinniefalco/boost-system/teams",
"hooks_url": "https://api.github.com/repos/vinniefalco/boost-system/hooks",
"issue_events_url": "https://api.github.com/repos/vinniefalco/boost-system/issues/events{/number}",
"events_url": "https://api.github.com/repos/vinniefalco/boost-system/events",
"assignees_url": "https://api.github.com/repos/vinniefalco/boost-system/assignees{/user}",
"branches_url": "https://api.github.com/repos/vinniefalco/boost-system/branches{/branch}",
"tags_url": "https://api.github.com/repos/vinniefalco/boost-system/tags",
"blobs_url": "https://api.github.com/repos/vinniefalco/boost-system/git/blobs{/sha}",
"git_tags_url": "https://api.github.com/repos/vinniefalco/boost-system/git/tags{/sha}",
"git_refs_url": "https://api.github.com/repos/vinniefalco/boost-system/git/refs{/sha}",
"trees_url": "https://api.github.com/repos/vinniefalco/boost-system/git/trees{/sha}",
"statuses_url": "https://api.github.com/repos/vinniefalco/boost-system/statuses/{sha}",
"languages_url": "https://api.github.com/repos/vinniefalco/boost-system/languages",
"stargazers_url": "https://api.github.com/repos/vinniefalco/boost-system/stargazers",
"contributors_url": "https://api.github.com/repos/vinniefalco/boost-system/contributors",
"subscribers_url": "https://api.github.com/repos/vinniefalco/boost-system/subscribers",
"subscription_url": "https://api.github.com/repos/vinniefalco/boost-system/subscription",
"commits_url": "https://api.github.com/repos/vinniefalco/boost-system/commits{/sha}",
"git_commits_url": "https://api.github.com/repos/vinniefalco/boost-system/git/commits{/sha}",
"comments_url": "https://api.github.com/repos/vinniefalco/boost-system/comments{/number}",
"issue_comment_url": "https://api.github.com/repos/vinniefalco/boost-system/issues/comments{/number}",
"contents_url": "https://api.github.com/repos/vinniefalco/boost-system/contents/{+path}",
"compare_url": "https://api.github.com/repos/vinniefalco/boost-system/compare/{base}...{head}",
"merges_url": "https://api.github.com/repos/vinniefalco/boost-system/merges",
"archive_url": "https://api.github.com/repos/vinniefalco/boost-system/{archive_format}{/ref}",
"downloads_url": "https://api.github.com/repos/vinniefalco/boost-system/downloads",
"issues_url": "https://api.github.com/repos/vinniefalco/boost-system/issues{/number}",
"pulls_url": "https://api.github.com/repos/vinniefalco/boost-system/pulls{/number}",
"milestones_url": "https://api.github.com/repos/vinniefalco/boost-system/milestones{/number}",
"notifications_url": "https://api.github.com/repos/vinniefalco/boost-system/notifications{?since,all,participating}",
"labels_url": "https://api.github.com/repos/vinniefalco/boost-system/labels{/name}",
"releases_url": "https://api.github.com/repos/vinniefalco/boost-system/releases{/id}",
"deployments_url": "https://api.github.com/repos/vinniefalco/boost-system/deployments",
"created_at": "2022-08-18T23:48:50Z",
"updated_at": "2022-08-19T01:05:39Z",
"pushed_at": "2022-08-21T22:22:12Z",
"git_url": "git://github.com/vinniefalco/boost-system.git",
"ssh_url": "git@github.com:vinniefalco/boost-system.git",
"clone_url": "https://github.com/vinniefalco/boost-system.git",
"svn_url": "https://github.com/vinniefalco/boost-system",
"homepage": "http://boost.org/libs/system",
"size": 781,
"stargazers_count": 0,
"watchers_count": 0,
"language": "C++",
"has_issues": False,
"has_projects": True,
"has_downloads": True,
"has_wiki": True,
"has_pages": False,
"has_discussions": False,
"forks_count": 0,
"mirror_url": None,
"archived": False,
"disabled": False,
"open_issues_count": 0,
"license": None,
"allow_forking": True,
"is_template": False,
"web_commit_signoff_required": False,
"topics": [],
"visibility": "public",
"forks": 0,
"open_issues": 0,
"watchers": 0,
"default_branch": "develop",
},
},
"base": {
"label": "boostorg:develop",
"ref": "develop",
"sha": "8c740705e6a221ef5fed7402338ba475df84077d",
"user": {
"login": "boostorg",
"id": 3170529,
"node_id": "MDEyOk9yZ2FuaXphdGlvbjMxNzA1Mjk=",
"avatar_url": "https://avatars.githubusercontent.com/u/3170529?v=4",
"gravatar_id": "",
"url": "https://api.github.com/users/boostorg",
"html_url": "https://github.com/boostorg",
"followers_url": "https://api.github.com/users/boostorg/followers",
"following_url": "https://api.github.com/users/boostorg/following{/other_user}",
"gists_url": "https://api.github.com/users/boostorg/gists{/gist_id}",
"starred_url": "https://api.github.com/users/boostorg/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/boostorg/subscriptions",
"organizations_url": "https://api.github.com/users/boostorg/orgs",
"repos_url": "https://api.github.com/users/boostorg/repos",
"events_url": "https://api.github.com/users/boostorg/events{/privacy}",
"received_events_url": "https://api.github.com/users/boostorg/received_events",
"type": "Organization",
"site_admin": False,
},
"repo": {
"id": 7590027,
"node_id": "MDEwOlJlcG9zaXRvcnk3NTkwMDI3",
"name": "system",
"full_name": "boostorg/system",
"private": False,
"owner": {
"login": "boostorg",
"id": 3170529,
"node_id": "MDEyOk9yZ2FuaXphdGlvbjMxNzA1Mjk=",
"avatar_url": "https://avatars.githubusercontent.com/u/3170529?v=4",
"gravatar_id": "",
"url": "https://api.github.com/users/boostorg",
"html_url": "https://github.com/boostorg",
"followers_url": "https://api.github.com/users/boostorg/followers",
"following_url": "https://api.github.com/users/boostorg/following{/other_user}",
"gists_url": "https://api.github.com/users/boostorg/gists{/gist_id}",
"starred_url": "https://api.github.com/users/boostorg/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/boostorg/subscriptions",
"organizations_url": "https://api.github.com/users/boostorg/orgs",
"repos_url": "https://api.github.com/users/boostorg/repos",
"events_url": "https://api.github.com/users/boostorg/events{/privacy}",
"received_events_url": "https://api.github.com/users/boostorg/received_events",
"type": "Organization",
"site_admin": False,
},
"html_url": "https://github.com/boostorg/system",
"description": "Boost.org system module ",
"fork": False,
"url": "https://api.github.com/repos/boostorg/system",
"forks_url": "https://api.github.com/repos/boostorg/system/forks",
"keys_url": "https://api.github.com/repos/boostorg/system/keys{/key_id}",
"collaborators_url": "https://api.github.com/repos/boostorg/system/collaborators{/collaborator}",
"teams_url": "https://api.github.com/repos/boostorg/system/teams",
"hooks_url": "https://api.github.com/repos/boostorg/system/hooks",
"issue_events_url": "https://api.github.com/repos/boostorg/system/issues/events{/number}",
"events_url": "https://api.github.com/repos/boostorg/system/events",
"assignees_url": "https://api.github.com/repos/boostorg/system/assignees{/user}",
"branches_url": "https://api.github.com/repos/boostorg/system/branches{/branch}",
"tags_url": "https://api.github.com/repos/boostorg/system/tags",
"blobs_url": "https://api.github.com/repos/boostorg/system/git/blobs{/sha}",
"git_tags_url": "https://api.github.com/repos/boostorg/system/git/tags{/sha}",
"git_refs_url": "https://api.github.com/repos/boostorg/system/git/refs{/sha}",
"trees_url": "https://api.github.com/repos/boostorg/system/git/trees{/sha}",
"statuses_url": "https://api.github.com/repos/boostorg/system/statuses/{sha}",
"languages_url": "https://api.github.com/repos/boostorg/system/languages",
"stargazers_url": "https://api.github.com/repos/boostorg/system/stargazers",
"contributors_url": "https://api.github.com/repos/boostorg/system/contributors",
"subscribers_url": "https://api.github.com/repos/boostorg/system/subscribers",
"subscription_url": "https://api.github.com/repos/boostorg/system/subscription",
"commits_url": "https://api.github.com/repos/boostorg/system/commits{/sha}",
"git_commits_url": "https://api.github.com/repos/boostorg/system/git/commits{/sha}",
"comments_url": "https://api.github.com/repos/boostorg/system/comments{/number}",
"issue_comment_url": "https://api.github.com/repos/boostorg/system/issues/comments{/number}",
"contents_url": "https://api.github.com/repos/boostorg/system/contents/{+path}",
"compare_url": "https://api.github.com/repos/boostorg/system/compare/{base}...{head}",
"merges_url": "https://api.github.com/repos/boostorg/system/merges",
"archive_url": "https://api.github.com/repos/boostorg/system/{archive_format}{/ref}",
"downloads_url": "https://api.github.com/repos/boostorg/system/downloads",
"issues_url": "https://api.github.com/repos/boostorg/system/issues{/number}",
"pulls_url": "https://api.github.com/repos/boostorg/system/pulls{/number}",
"milestones_url": "https://api.github.com/repos/boostorg/system/milestones{/number}",
"notifications_url": "https://api.github.com/repos/boostorg/system/notifications{?since,all,participating}",
"labels_url": "https://api.github.com/repos/boostorg/system/labels{/name}",
"releases_url": "https://api.github.com/repos/boostorg/system/releases{/id}",
"deployments_url": "https://api.github.com/repos/boostorg/system/deployments",
"created_at": "2013-01-13T15:59:31Z",
"updated_at": "2022-12-14T22:25:46Z",
"pushed_at": "2022-12-14T15:17:31Z",
"git_url": "git://github.com/boostorg/system.git",
"ssh_url": "git@github.com:boostorg/system.git",
"clone_url": "https://github.com/boostorg/system.git",
"svn_url": "https://github.com/boostorg/system",
"homepage": "http://boost.org/libs/system",
"size": 852,
"stargazers_count": 26,
"watchers_count": 26,
"language": "C++",
"has_issues": True,
"has_projects": False,
"has_downloads": True,
"has_wiki": False,
"has_pages": False,
"has_discussions": False,
"forks_count": 82,
"mirror_url": None,
"archived": False,
"disabled": False,
"open_issues_count": 10,
"license": None,
"allow_forking": True,
"is_template": False,
"web_commit_signoff_required": False,
"topics": [],
"visibility": "public",
"forks": 82,
"open_issues": 10,
"watchers": 26,
"default_branch": "develop",
},
},
"_links": {
"self": {"href": "https://api.github.com/repos/boostorg/system/pulls/90"},
"html": {"href": "https://github.com/boostorg/system/pull/90"},
"issue": {"href": "https://api.github.com/repos/boostorg/system/issues/90"},
"comments": {
"href": "https://api.github.com/repos/boostorg/system/issues/90/comments"
},
"review_comments": {
"href": "https://api.github.com/repos/boostorg/system/pulls/90/comments"
},
"review_comment": {
"href": "https://api.github.com/repos/boostorg/system/pulls/comments{/number}"
},
"commits": {
"href": "https://api.github.com/repos/boostorg/system/pulls/90/commits"
},
"statuses": {
"href": "https://api.github.com/repos/boostorg/system/statuses/fe48c3058daaa31da6c50c316d63aa5f185dacb8"
},
},
"author_association": "MEMBER",
"auto_merge": None,
"active_lock_reason": None,
}
return [
dict2obj(
{
"title": "Improve logging",
"number": 1,
"state": "closed",
"closed_at": "2022-04-11T12:38:24Z",
"merged_at": "2022-04-11T12:38:24Z",
"created_at": "2022-04-11T11:41:02Z",
"updated_at": "2022-04-11T12:38:25Z",
"id": 5898798798,
}
),
dict2obj(
{
"title": "Fix a test",
"number": 2,
"state": "open",
"closed_at": "2022-04-11T12:38:24Z",
"merged_at": "2022-04-11T12:38:24Z",
"created_at": "2022-04-11T11:41:02Z",
"updated_at": "2022-04-11T12:38:25Z",
"id": 7395968281,
}
),
dict2obj(
{
"title": "Add a new feature",
"number": 3,
"state": "closed",
"closed_at": "2022-04-11T12:38:24Z",
"merged_at": "2022-04-11T12:38:24Z",
"created_at": "2022-04-11T11:41:02Z",
"updated_at": "2022-04-11T12:38:25Z",
"id": 7492027464,
}
),
]
@pytest.fixture

159
libraries/tests/test_github.py
View File

@@ -1,10 +1,13 @@
from unittest.mock import patch
import pytest
import responses
from dateutil.parser import parse
from ghapi.all import GhApi
from model_bakery import baker
from libraries.github import LibraryUpdater, get_api
from libraries.models import Library
from libraries.github import GithubUpdater, LibraryUpdater, get_api
from libraries.models import Issue, Library, PullRequest
def test_get_api():
@@ -12,6 +15,158 @@ def test_get_api():
assert isinstance(result, GhApi)
# GithubUpdater tests
def test_update_issues_new(tp, library, github_api_repo_issues_response):
"""GithubUpdater.update_issues()"""
new_issues_count = len(github_api_repo_issues_response)
expected_count = Issue.objects.count() + new_issues_count
with patch("libraries.github.repo_issues") as repo_issues_mock:
updater = GithubUpdater(library=library)
repo_issues_mock.return_value = github_api_repo_issues_response
updater.update_issues()
assert Issue.objects.count() == expected_count
ids = [issue.id for issue in github_api_repo_issues_response]
issues = Issue.objects.filter(library=library, github_id__in=ids)
assert issues.exists()
assert issues.count() == expected_count
# Test the values of a sample Issue
gh_issue = github_api_repo_issues_response[0]
issue = issues.get(github_id=gh_issue.id)
assert issue.title == gh_issue.title
assert issue.number == gh_issue.number
if gh_issue.state == "open":
assert issue.is_open
else:
assert not issue.is_open
assert issue.data == gh_issue
expected_closed = parse(gh_issue["closed_at"])
expected_created = parse(gh_issue["created_at"])
expected_modified = parse(gh_issue["updated_at"])
assert issue.closed == expected_closed
assert issue.created == expected_created
assert issue.modified == expected_modified
def test_update_issues_existing(tp, library, github_api_repo_issues_response):
"""Test that GithubUpdater.update_issues() updates existing issues when appropriate"""
existing_issue_data = github_api_repo_issues_response[0]
old_title = "Old title"
issue = baker.make(
Issue, library=library, github_id=existing_issue_data.id, title=old_title
)
# Make sure we are expected one fewer new issue, since we created one in advance
new_issues_count = len(github_api_repo_issues_response)
expected_count = Issue.objects.count() + new_issues_count - 1
with patch("libraries.github.repo_issues") as repo_issues_mock:
updater = GithubUpdater(library=library)
repo_issues_mock.return_value = github_api_repo_issues_response
updater.update_issues()
assert Issue.objects.count() == expected_count
ids = [issue.id for issue in github_api_repo_issues_response]
issues = Issue.objects.filter(library=library, github_id__in=ids)
assert issues.exists()
assert issues.count() == expected_count
# Test that the existing issue updated
issue.refresh_from_db()
assert issue.title == existing_issue_data.title
def test_update_issues_long_title(tp, library, github_api_repo_issues_response):
"""Test that GithubUpdater.update_issues() handles long title gracefully"""
new_issues_count = len(github_api_repo_issues_response)
expected_count = Issue.objects.count() + new_issues_count
title = "sample" * 100
assert len(title) > 255
expected_title = title[:255]
assert len(expected_title) <= 255
with patch("libraries.github.repo_issues") as repo_issues_mock:
updater = GithubUpdater(library=library)
# Make an extra-long title so we can confirm that it saves
github_id = github_api_repo_issues_response[0]["id"]
github_api_repo_issues_response[0]["title"] = "sample" * 100
repo_issues_mock.return_value = github_api_repo_issues_response
updater.update_issues()
assert Issue.objects.count() == expected_count
assert Issue.objects.filter(library=library, github_id=github_id).exists()
issue = Issue.objects.get(library=library, github_id=github_id)
assert issue.title == expected_title
def test_update_prs_new(tp, library, github_api_repo_prs_response):
"""Test that GithubUpdater.update_prs() imports new PRs appropriately"""
new_prs_count = len(github_api_repo_prs_response)
expected_count = PullRequest.objects.count() + new_prs_count
with patch("libraries.github.repo_prs") as repo_prs_mock:
updater = GithubUpdater(library=library)
github_api_repo_prs_response[0]["title"] = "sample" * 100
repo_prs_mock.return_value = github_api_repo_prs_response
updater.update_prs()
assert PullRequest.objects.count() == expected_count
ids = [pr.id for pr in github_api_repo_prs_response]
pulls = PullRequest.objects.filter(library=library, github_id__in=ids)
assert pulls.exists()
assert pulls.count() == expected_count
# Test the values of a sample PR
gh_pull = github_api_repo_prs_response[0]
pr = pulls.get(github_id=gh_pull.id)
assert pr.title == gh_pull.title[:255]
assert pr.number == gh_pull.number
if gh_pull.state == "open":
assert pr.is_open
else:
assert not pr.is_open
assert pr.data == gh_pull
expected_closed = parse(gh_pull["closed_at"])
expected_created = parse(gh_pull["created_at"])
expected_modified = parse(gh_pull["updated_at"])
assert pr.closed == expected_closed
assert pr.created == expected_created
assert pr.modified == expected_modified
def test_update_prs_existing(tp, library, github_api_repo_prs_response):
"""Test that GithubUpdater.update_prs() updates existing PRs when appropriate"""
existing_pr_data = github_api_repo_prs_response[0]
old_title = "Old title"
pull = baker.make(
PullRequest, library=library, github_id=existing_pr_data.id, title=old_title
)
# Make sure we are expected one fewer new PRs, since we created one in advance
new_prs_count = len(github_api_repo_prs_response)
expected_count = PullRequest.objects.count() + new_prs_count - 1
with patch("libraries.github.repo_prs") as repo_prs_mock:
updater = GithubUpdater(library=library)
repo_prs_mock.return_value = github_api_repo_prs_response
updater.update_prs()
assert PullRequest.objects.count() == expected_count
ids = [pr.id for pr in github_api_repo_prs_response]
pulls = PullRequest.objects.filter(library=library, github_id__in=ids)
assert pulls.exists()
assert pulls.count() == expected_count
# Test that the existing PR updated
pull.refresh_from_db()
assert pull.title == existing_pr_data.title
# LibraryUpdater tests

22
libraries/tests/test_utils.py Normal file
View File

@@ -0,0 +1,22 @@
from datetime import datetime
from libraries.utils import parse_date
def test_parse_date_iso():
expected = datetime.now()
result = parse_date(expected.isoformat())
assert expected == result
def test_parse_date_str():
expected = datetime.now()
input_date = f"{expected.month}-{expected.day}-{expected.year}"
result = parse_date(input_date)
assert expected.date() == result.date()
def test_parse_date_str_none():
expected = None
result = parse_date("")
assert expected == result

45
libraries/tests/test_views.py
View File

@@ -1,15 +1,50 @@
from unittest.mock import patch
from model_bakery import baker
def test_library_list(library, tp):
"""GET /libraries/"""
res = tp.get("libraries")
tp.response_200(res)
def test_library_detail(library, tp):
"""GET /libraries/{repo}/"""
url = tp.reverse("library-detail", library.slug)
response = tp.get(url)
tp.response_200(response)
with patch("libraries.views.LibraryDetail.get_open_issues_count") as count_mock:
count_mock.return_value = 21
res = tp.get(url)
tp.response_200(res)
def test_library_detail_context_get_closed_prs_count(tp, library):
"""
GET /libraries/{repo}/
Test that the custom closed_prs_count var appears as expected
"""
# Create open and closed PRs for this library, and another random PR
lib2 = baker.make("libraries.Library", slug="sample")
baker.make("libraries.PullRequest", library=library, is_open=True)
baker.make("libraries.PullRequest", library=library, is_open=False)
baker.make("libraries.PullRequest", library=lib2, is_open=True)
url = tp.reverse("library-detail", library.slug)
response = tp.get(url)
tp.response_200(response)
assert "closed_prs_count" in response.context
# Verify that the count only includes the one open PR for this library
assert response.context["closed_prs_count"] == 1
def test_library_detail_context_get_open_issues_count(tp, library):
"""
GET /libraries/{repo}/
Test that the custom open_issues_count var appears as expected
"""
# Create open and closed issues for this library, and another random issue
lib2 = baker.make("libraries.Library", slug="sample")
baker.make("libraries.Issue", library=library, is_open=True)
baker.make("libraries.Issue", library=library, is_open=False)
baker.make("libraries.Issue", library=lib2, is_open=True)
url = tp.reverse("library-detail", library.slug)
response = tp.get(url)
tp.response_200(response)
assert "open_issues_count" in response.context
# Verify that the count only includes the one open issue for this library
assert response.context["open_issues_count"] == 1

14
libraries/utils.py Normal file
View File

@@ -0,0 +1,14 @@
import structlog
from dateutil.parser import ParserError, parse
logger = structlog.get_logger()
def parse_date(date_str):
"""Parses a date string to a datetime. Does not return an error."""
try:
return parse(date_str)
except ParserError:
logger.info("parse_date_invalid_date", date_str=date_str)
return None

15
libraries/views.py
View File

@@ -1,7 +1,6 @@
from django.views.generic import DetailView, ListView
from .github import repo_issues
from .models import Category, Library
from .models import Category, Issue, Library, PullRequest
class CategoryMixin:
@@ -60,14 +59,12 @@ class LibraryDetail(CategoryMixin, DetailView):
def get(self, request, *args, **kwargs):
self.object = self.get_object()
context = self.get_context_data(object=self.object)
context["closed_prs_count"] = self.get_closed_prs_count(self.object)
context["open_issues_count"] = self.get_open_issues_count(self.object)
return self.render_to_response(context)
def get_closed_prs_count(self, obj):
return PullRequest.objects.filter(library=obj, is_open=True).count()
def get_open_issues_count(self, obj):
try:
issues = repo_issues(
obj.github_owner, obj.github_repo, state="open", issues_only=True
)
return len(issues)
except Exception:
return 0
return Issue.objects.filter(library=obj, is_open=True).count()

2
templates/libraries/detail.html
View File

@@ -63,7 +63,7 @@
<div class="md:flex my-4 md:my-11 py-6 md:justify-between">
<div class="py-6">
<h3 class="text-2xl mb-1">Closed Pull Requests</h3>
X per Month
{{ closed_prs_count }}
</div>
<div class="py-6">