diff --git a/config/test_settings.py b/config/test_settings.py index 73b3419e..1dfdcf2c 100644 --- a/config/test_settings.py +++ b/config/test_settings.py @@ -25,3 +25,5 @@ MIGRATION_MODULES = DisableMigrations() # User a faster password hasher PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] + +GITHUB_TOKEN = "changeme" diff --git a/libraries/github.py b/libraries/github.py index a62441a6..1026132d 100644 --- a/libraries/github.py +++ b/libraries/github.py @@ -1,12 +1,11 @@ import base64 import os import re - import requests import structlog -from dateutil.parser import ParserError, parse -from ghapi.all import GhApi, paged +from fastcore.xtras import obj2dict +from ghapi.all import GhApi, paged from .models import Category, Issue, Library, PullRequest from .utils import parse_date @@ -37,6 +36,9 @@ def repo_issues(owner, repo, state="all", issues_only=True): Note: The GitHub API considers both PRs and Issues to be "Issues" and does not support filtering in the request, so to exclude PRs from the list of issues, we do some manual filtering of the results + + Note: GhApi() returns results as AttrDict objects: + https://fastcore.fast.ai/basics.html#attrdict """ api = get_api() pages = list( @@ -64,7 +66,12 @@ def repo_issues(owner, repo, state="all", issues_only=True): def repo_prs(owner, repo, state="all"): - """Get all PRs for a repo""" + """ + Get all PRs for a repo + + Note: GhApi() returns results as AttrDict objects: + https://fastcore.fast.ai/basics.html#attrdict + """ api = get_api() pages = list( paged( @@ -192,11 +199,7 @@ class LibraryUpdater: meta = self.get_library_metadata(repo=name) github_data = self.get_library_github_data(owner=self.owner, repo=name) - - try: - last_github_update = parse(github_data.get("updated_at")) - except ParserError: - last_github_update = None + last_github_update = parse_date(github_data.get("updated_at", "")) github_url = f"https://github.com/boostorg/{name}/" if type(meta) is list: @@ -262,7 +265,9 @@ class LibraryUpdater: self.logger.info("update_all_libraries_metadata", library_count=len(libs)) for lib in libs: - self.update_library(lib) + library = self.update_library(lib) + github_updater = GithubUpdater(owner=self.owner, library=library) + github_updater.update() def update_categories(self, obj, categories): """Update all of the categories for an object""" @@ -291,6 +296,8 @@ class LibraryUpdater: logger.info("library_udpated") + return obj + except Exception: logger.exception("library_update_failed") @@ -302,13 +309,12 @@ class GithubUpdater: for the site """ - def __init__(self, owner, repo): + def __init__(self, owner="boostorg", library=None): self.owner = owner - self.repo = repo - self.logger = logger.bind(owner=owner, repo=repo) + self.library = library + self.logger = logger.bind(owner=owner, library=library) def update(self): - # FIXME: Write a test self.logger.info("update_github_repo") try: @@ -322,6 +328,7 @@ class GithubUpdater: self.logger.exception("update_prs_error") def update_issues(self): + """Update all issues for a library""" self.logger.info("updating_repo_issues") issues_data = repo_issues( @@ -365,11 +372,10 @@ class GithubUpdater: issue_github_id=issue_dict.get("id"), exc_msg=str(e), ) - continue logger.info( "issue_updated_successfully", issue_id=issue.id, - created=created, + created_issue=created, issue_github_id=issue.github_id, ) @@ -377,9 +383,10 @@ class GithubUpdater: """Update all PRs for a library""" self.logger.info("updating_repo_prs") - prs_data = repo_prs(self.owner, self.library.name) + prs_data = repo_prs(self.owner, self.library.name, state="all") for pr_dict in prs_data: + # Get the date information closed_at = None merged_at = None @@ -395,10 +402,8 @@ class GithubUpdater: if pr_dict.get("created_at"): created_at = parse_date(pr_dict["created_at"]) - if pr_dict.get("modified_at"): - modified_at = parse_date(pr_dict["modified_at"]) - - breakpoint() + if pr_dict.get("updated_at"): + modified_at = parse_date(pr_dict["updated_at"]) try: pull_request, created = PullRequest.objects.update_or_create( @@ -421,11 +426,9 @@ class GithubUpdater: pr_github_id=pr_dict.get("id"), exc_msg=str(e), ) - continue - logger.info( - "pr_updated_successfully", - pull_request_id=pull_request.id, - created=created, - pull_request_github_id=pull_request.github_id, + "pull_request_updated_successfully", + pr_id=pull_request.id, + created_pr=created, + pr_github_id=pull_request.github_id, ) diff --git a/libraries/management/commands/update_libraries.py b/libraries/management/commands/update_libraries.py index 78f21c3f..1337cd9f 100644 --- a/libraries/management/commands/update_libraries.py +++ b/libraries/management/commands/update_libraries.py @@ -5,5 +5,11 @@ from libraries.github import LibraryUpdater @click.command() def command(): - l = LibraryUpdater() - l.update_libraries() + """ + Calls the LibraryUpdater, which retrieves the active boost libraries + from the Boost repo and updates the models in our database with the latest + information on that library (repo) and its issues, pull requests, and related + objects from GitHub. + """ + updater = LibraryUpdater() + updater.update_libraries() diff --git a/libraries/tests/fixtures.py b/libraries/tests/fixtures.py index 259a3000..a8a85bc2 100644 --- a/libraries/tests/fixtures.py +++ b/libraries/tests/fixtures.py @@ -1,4 +1,5 @@ import pytest +from fastcore.xtras import dict2obj from model_bakery import baker @@ -103,330 +104,44 @@ def github_api_repo_issues_response(db): @pytest.fixture def github_api_repo_prs_response(db): """Returns the response from GhApi().pulls.list, already paged""" - return { - "url": "https://api.github.com/repos/boostorg/system/pulls/90", - "id": 1032140532, - "node_id": "PR_kwDOAHPQi849hTb0", - "html_url": "https://github.com/boostorg/system/pull/90", - "diff_url": "https://github.com/boostorg/system/pull/90.diff", - "patch_url": "https://github.com/boostorg/system/pull/90.patch", - "issue_url": "https://api.github.com/repos/boostorg/system/issues/90", - "number": 90, - "state": "open", - "locked": False, - "title": "add boost_system.natvis and interface source files", - "user": { - "login": "vinniefalco", - "id": 1503976, - "node_id": "MDQ6VXNlcjE1MDM5NzY=", - "avatar_url": "https://avatars.githubusercontent.com/u/1503976?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/vinniefalco", - "html_url": "https://github.com/vinniefalco", - "followers_url": "https://api.github.com/users/vinniefalco/followers", - "following_url": "https://api.github.com/users/vinniefalco/following{/other_user}", - "gists_url": "https://api.github.com/users/vinniefalco/gists{/gist_id}", - "starred_url": "https://api.github.com/users/vinniefalco/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/vinniefalco/subscriptions", - "organizations_url": "https://api.github.com/users/vinniefalco/orgs", - "repos_url": "https://api.github.com/users/vinniefalco/repos", - "events_url": "https://api.github.com/users/vinniefalco/events{/privacy}", - "received_events_url": "https://api.github.com/users/vinniefalco/received_events", - "type": "User", - "site_admin": False, - }, - "body": None, - "created_at": "2022-08-21T22:24:43Z", - "updated_at": "2022-08-21T22:24:43Z", - "closed_at": None, - "merged_at": None, - "merge_commit_sha": "37653ac206475a046d7e7abadaf823430e564572", - "assignee": None, - "assignees": [], - "requested_reviewers": [], - "requested_teams": [], - "labels": [], - "milestone": None, - "draft": False, - "commits_url": "https://api.github.com/repos/boostorg/system/pulls/90/commits", - "review_comments_url": "https://api.github.com/repos/boostorg/system/pulls/90/comments", - "review_comment_url": "https://api.github.com/repos/boostorg/system/pulls/comments{/number}", - "comments_url": "https://api.github.com/repos/boostorg/system/issues/90/comments", - "statuses_url": "https://api.github.com/repos/boostorg/system/statuses/fe48c3058daaa31da6c50c316d63aa5f185dacb8", - "head": { - "label": "vinniefalco:natvis", - "ref": "natvis", - "sha": "fe48c3058daaa31da6c50c316d63aa5f185dacb8", - "user": { - "login": "vinniefalco", - "id": 1503976, - "node_id": "MDQ6VXNlcjE1MDM5NzY=", - "avatar_url": "https://avatars.githubusercontent.com/u/1503976?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/vinniefalco", - "html_url": "https://github.com/vinniefalco", - "followers_url": "https://api.github.com/users/vinniefalco/followers", - "following_url": "https://api.github.com/users/vinniefalco/following{/other_user}", - "gists_url": "https://api.github.com/users/vinniefalco/gists{/gist_id}", - "starred_url": "https://api.github.com/users/vinniefalco/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/vinniefalco/subscriptions", - "organizations_url": "https://api.github.com/users/vinniefalco/orgs", - "repos_url": "https://api.github.com/users/vinniefalco/repos", - "events_url": "https://api.github.com/users/vinniefalco/events{/privacy}", - "received_events_url": "https://api.github.com/users/vinniefalco/received_events", - "type": "User", - "site_admin": False, - }, - "repo": { - "id": 526406204, - "node_id": "R_kgDOH2BSPA", - "name": "boost-system", - "full_name": "vinniefalco/boost-system", - "private": False, - "owner": { - "login": "vinniefalco", - "id": 1503976, - "node_id": "MDQ6VXNlcjE1MDM5NzY=", - "avatar_url": "https://avatars.githubusercontent.com/u/1503976?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/vinniefalco", - "html_url": "https://github.com/vinniefalco", - "followers_url": "https://api.github.com/users/vinniefalco/followers", - "following_url": "https://api.github.com/users/vinniefalco/following{/other_user}", - "gists_url": "https://api.github.com/users/vinniefalco/gists{/gist_id}", - "starred_url": "https://api.github.com/users/vinniefalco/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/vinniefalco/subscriptions", - "organizations_url": "https://api.github.com/users/vinniefalco/orgs", - "repos_url": "https://api.github.com/users/vinniefalco/repos", - "events_url": "https://api.github.com/users/vinniefalco/events{/privacy}", - "received_events_url": "https://api.github.com/users/vinniefalco/received_events", - "type": "User", - "site_admin": False, - }, - "html_url": "https://github.com/vinniefalco/boost-system", - "description": "Boost.org system module ", - "fork": True, - "url": "https://api.github.com/repos/vinniefalco/boost-system", - "forks_url": "https://api.github.com/repos/vinniefalco/boost-system/forks", - "keys_url": "https://api.github.com/repos/vinniefalco/boost-system/keys{/key_id}", - "collaborators_url": "https://api.github.com/repos/vinniefalco/boost-system/collaborators{/collaborator}", - "teams_url": "https://api.github.com/repos/vinniefalco/boost-system/teams", - "hooks_url": "https://api.github.com/repos/vinniefalco/boost-system/hooks", - "issue_events_url": "https://api.github.com/repos/vinniefalco/boost-system/issues/events{/number}", - "events_url": "https://api.github.com/repos/vinniefalco/boost-system/events", - "assignees_url": "https://api.github.com/repos/vinniefalco/boost-system/assignees{/user}", - "branches_url": "https://api.github.com/repos/vinniefalco/boost-system/branches{/branch}", - "tags_url": "https://api.github.com/repos/vinniefalco/boost-system/tags", - "blobs_url": "https://api.github.com/repos/vinniefalco/boost-system/git/blobs{/sha}", - "git_tags_url": "https://api.github.com/repos/vinniefalco/boost-system/git/tags{/sha}", - "git_refs_url": "https://api.github.com/repos/vinniefalco/boost-system/git/refs{/sha}", - "trees_url": "https://api.github.com/repos/vinniefalco/boost-system/git/trees{/sha}", - "statuses_url": "https://api.github.com/repos/vinniefalco/boost-system/statuses/{sha}", - "languages_url": "https://api.github.com/repos/vinniefalco/boost-system/languages", - "stargazers_url": "https://api.github.com/repos/vinniefalco/boost-system/stargazers", - "contributors_url": "https://api.github.com/repos/vinniefalco/boost-system/contributors", - "subscribers_url": "https://api.github.com/repos/vinniefalco/boost-system/subscribers", - "subscription_url": "https://api.github.com/repos/vinniefalco/boost-system/subscription", - "commits_url": "https://api.github.com/repos/vinniefalco/boost-system/commits{/sha}", - "git_commits_url": "https://api.github.com/repos/vinniefalco/boost-system/git/commits{/sha}", - "comments_url": "https://api.github.com/repos/vinniefalco/boost-system/comments{/number}", - "issue_comment_url": "https://api.github.com/repos/vinniefalco/boost-system/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/vinniefalco/boost-system/contents/{+path}", - "compare_url": "https://api.github.com/repos/vinniefalco/boost-system/compare/{base}...{head}", - "merges_url": "https://api.github.com/repos/vinniefalco/boost-system/merges", - "archive_url": "https://api.github.com/repos/vinniefalco/boost-system/{archive_format}{/ref}", - "downloads_url": "https://api.github.com/repos/vinniefalco/boost-system/downloads", - "issues_url": "https://api.github.com/repos/vinniefalco/boost-system/issues{/number}", - "pulls_url": "https://api.github.com/repos/vinniefalco/boost-system/pulls{/number}", - "milestones_url": "https://api.github.com/repos/vinniefalco/boost-system/milestones{/number}", - "notifications_url": "https://api.github.com/repos/vinniefalco/boost-system/notifications{?since,all,participating}", - "labels_url": "https://api.github.com/repos/vinniefalco/boost-system/labels{/name}", - "releases_url": "https://api.github.com/repos/vinniefalco/boost-system/releases{/id}", - "deployments_url": "https://api.github.com/repos/vinniefalco/boost-system/deployments", - "created_at": "2022-08-18T23:48:50Z", - "updated_at": "2022-08-19T01:05:39Z", - "pushed_at": "2022-08-21T22:22:12Z", - "git_url": "git://github.com/vinniefalco/boost-system.git", - "ssh_url": "git@github.com:vinniefalco/boost-system.git", - "clone_url": "https://github.com/vinniefalco/boost-system.git", - "svn_url": "https://github.com/vinniefalco/boost-system", - "homepage": "http://boost.org/libs/system", - "size": 781, - "stargazers_count": 0, - "watchers_count": 0, - "language": "C++", - "has_issues": False, - "has_projects": True, - "has_downloads": True, - "has_wiki": True, - "has_pages": False, - "has_discussions": False, - "forks_count": 0, - "mirror_url": None, - "archived": False, - "disabled": False, - "open_issues_count": 0, - "license": None, - "allow_forking": True, - "is_template": False, - "web_commit_signoff_required": False, - "topics": [], - "visibility": "public", - "forks": 0, - "open_issues": 0, - "watchers": 0, - "default_branch": "develop", - }, - }, - "base": { - "label": "boostorg:develop", - "ref": "develop", - "sha": "8c740705e6a221ef5fed7402338ba475df84077d", - "user": { - "login": "boostorg", - "id": 3170529, - "node_id": "MDEyOk9yZ2FuaXphdGlvbjMxNzA1Mjk=", - "avatar_url": "https://avatars.githubusercontent.com/u/3170529?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/boostorg", - "html_url": "https://github.com/boostorg", - "followers_url": "https://api.github.com/users/boostorg/followers", - "following_url": "https://api.github.com/users/boostorg/following{/other_user}", - "gists_url": "https://api.github.com/users/boostorg/gists{/gist_id}", - "starred_url": "https://api.github.com/users/boostorg/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/boostorg/subscriptions", - "organizations_url": "https://api.github.com/users/boostorg/orgs", - "repos_url": "https://api.github.com/users/boostorg/repos", - "events_url": "https://api.github.com/users/boostorg/events{/privacy}", - "received_events_url": "https://api.github.com/users/boostorg/received_events", - "type": "Organization", - "site_admin": False, - }, - "repo": { - "id": 7590027, - "node_id": "MDEwOlJlcG9zaXRvcnk3NTkwMDI3", - "name": "system", - "full_name": "boostorg/system", - "private": False, - "owner": { - "login": "boostorg", - "id": 3170529, - "node_id": "MDEyOk9yZ2FuaXphdGlvbjMxNzA1Mjk=", - "avatar_url": "https://avatars.githubusercontent.com/u/3170529?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/boostorg", - "html_url": "https://github.com/boostorg", - "followers_url": "https://api.github.com/users/boostorg/followers", - "following_url": "https://api.github.com/users/boostorg/following{/other_user}", - "gists_url": "https://api.github.com/users/boostorg/gists{/gist_id}", - "starred_url": "https://api.github.com/users/boostorg/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/boostorg/subscriptions", - "organizations_url": "https://api.github.com/users/boostorg/orgs", - "repos_url": "https://api.github.com/users/boostorg/repos", - "events_url": "https://api.github.com/users/boostorg/events{/privacy}", - "received_events_url": "https://api.github.com/users/boostorg/received_events", - "type": "Organization", - "site_admin": False, - }, - "html_url": "https://github.com/boostorg/system", - "description": "Boost.org system module ", - "fork": False, - "url": "https://api.github.com/repos/boostorg/system", - "forks_url": "https://api.github.com/repos/boostorg/system/forks", - "keys_url": "https://api.github.com/repos/boostorg/system/keys{/key_id}", - "collaborators_url": "https://api.github.com/repos/boostorg/system/collaborators{/collaborator}", - "teams_url": "https://api.github.com/repos/boostorg/system/teams", - "hooks_url": "https://api.github.com/repos/boostorg/system/hooks", - "issue_events_url": "https://api.github.com/repos/boostorg/system/issues/events{/number}", - "events_url": "https://api.github.com/repos/boostorg/system/events", - "assignees_url": "https://api.github.com/repos/boostorg/system/assignees{/user}", - "branches_url": "https://api.github.com/repos/boostorg/system/branches{/branch}", - "tags_url": "https://api.github.com/repos/boostorg/system/tags", - "blobs_url": "https://api.github.com/repos/boostorg/system/git/blobs{/sha}", - "git_tags_url": "https://api.github.com/repos/boostorg/system/git/tags{/sha}", - "git_refs_url": "https://api.github.com/repos/boostorg/system/git/refs{/sha}", - "trees_url": "https://api.github.com/repos/boostorg/system/git/trees{/sha}", - "statuses_url": "https://api.github.com/repos/boostorg/system/statuses/{sha}", - "languages_url": "https://api.github.com/repos/boostorg/system/languages", - "stargazers_url": "https://api.github.com/repos/boostorg/system/stargazers", - "contributors_url": "https://api.github.com/repos/boostorg/system/contributors", - "subscribers_url": "https://api.github.com/repos/boostorg/system/subscribers", - "subscription_url": "https://api.github.com/repos/boostorg/system/subscription", - "commits_url": "https://api.github.com/repos/boostorg/system/commits{/sha}", - "git_commits_url": "https://api.github.com/repos/boostorg/system/git/commits{/sha}", - "comments_url": "https://api.github.com/repos/boostorg/system/comments{/number}", - "issue_comment_url": "https://api.github.com/repos/boostorg/system/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/boostorg/system/contents/{+path}", - "compare_url": "https://api.github.com/repos/boostorg/system/compare/{base}...{head}", - "merges_url": "https://api.github.com/repos/boostorg/system/merges", - "archive_url": "https://api.github.com/repos/boostorg/system/{archive_format}{/ref}", - "downloads_url": "https://api.github.com/repos/boostorg/system/downloads", - "issues_url": "https://api.github.com/repos/boostorg/system/issues{/number}", - "pulls_url": "https://api.github.com/repos/boostorg/system/pulls{/number}", - "milestones_url": "https://api.github.com/repos/boostorg/system/milestones{/number}", - "notifications_url": "https://api.github.com/repos/boostorg/system/notifications{?since,all,participating}", - "labels_url": "https://api.github.com/repos/boostorg/system/labels{/name}", - "releases_url": "https://api.github.com/repos/boostorg/system/releases{/id}", - "deployments_url": "https://api.github.com/repos/boostorg/system/deployments", - "created_at": "2013-01-13T15:59:31Z", - "updated_at": "2022-12-14T22:25:46Z", - "pushed_at": "2022-12-14T15:17:31Z", - "git_url": "git://github.com/boostorg/system.git", - "ssh_url": "git@github.com:boostorg/system.git", - "clone_url": "https://github.com/boostorg/system.git", - "svn_url": "https://github.com/boostorg/system", - "homepage": "http://boost.org/libs/system", - "size": 852, - "stargazers_count": 26, - "watchers_count": 26, - "language": "C++", - "has_issues": True, - "has_projects": False, - "has_downloads": True, - "has_wiki": False, - "has_pages": False, - "has_discussions": False, - "forks_count": 82, - "mirror_url": None, - "archived": False, - "disabled": False, - "open_issues_count": 10, - "license": None, - "allow_forking": True, - "is_template": False, - "web_commit_signoff_required": False, - "topics": [], - "visibility": "public", - "forks": 82, - "open_issues": 10, - "watchers": 26, - "default_branch": "develop", - }, - }, - "_links": { - "self": {"href": "https://api.github.com/repos/boostorg/system/pulls/90"}, - "html": {"href": "https://github.com/boostorg/system/pull/90"}, - "issue": {"href": "https://api.github.com/repos/boostorg/system/issues/90"}, - "comments": { - "href": "https://api.github.com/repos/boostorg/system/issues/90/comments" - }, - "review_comments": { - "href": "https://api.github.com/repos/boostorg/system/pulls/90/comments" - }, - "review_comment": { - "href": "https://api.github.com/repos/boostorg/system/pulls/comments{/number}" - }, - "commits": { - "href": "https://api.github.com/repos/boostorg/system/pulls/90/commits" - }, - "statuses": { - "href": "https://api.github.com/repos/boostorg/system/statuses/fe48c3058daaa31da6c50c316d63aa5f185dacb8" - }, - }, - "author_association": "MEMBER", - "auto_merge": None, - "active_lock_reason": None, - } + return [ + dict2obj( + { + "title": "Improve logging", + "number": 1, + "state": "closed", + "closed_at": "2022-04-11T12:38:24Z", + "merged_at": "2022-04-11T12:38:24Z", + "created_at": "2022-04-11T11:41:02Z", + "updated_at": "2022-04-11T12:38:25Z", + "id": 5898798798, + } + ), + dict2obj( + { + "title": "Fix a test", + "number": 2, + "state": "open", + "closed_at": "2022-04-11T12:38:24Z", + "merged_at": "2022-04-11T12:38:24Z", + "created_at": "2022-04-11T11:41:02Z", + "updated_at": "2022-04-11T12:38:25Z", + "id": 7395968281, + } + ), + dict2obj( + { + "title": "Add a new feature", + "number": 3, + "state": "closed", + "closed_at": "2022-04-11T12:38:24Z", + "merged_at": "2022-04-11T12:38:24Z", + "created_at": "2022-04-11T11:41:02Z", + "updated_at": "2022-04-11T12:38:25Z", + "id": 7492027464, + } + ), + ] @pytest.fixture diff --git a/libraries/tests/test_github.py b/libraries/tests/test_github.py index 95be3a08..fe88aff8 100644 --- a/libraries/tests/test_github.py +++ b/libraries/tests/test_github.py @@ -1,10 +1,13 @@ from unittest.mock import patch +import pytest import responses +from dateutil.parser import parse from ghapi.all import GhApi +from model_bakery import baker -from libraries.github import LibraryUpdater, get_api -from libraries.models import Library +from libraries.github import GithubUpdater, LibraryUpdater, get_api +from libraries.models import Issue, Library, PullRequest def test_get_api(): @@ -12,6 +15,158 @@ def test_get_api(): assert isinstance(result, GhApi) +# GithubUpdater tests + + +def test_update_issues_new(tp, library, github_api_repo_issues_response): + """GithubUpdater.update_issues()""" + new_issues_count = len(github_api_repo_issues_response) + expected_count = Issue.objects.count() + new_issues_count + with patch("libraries.github.repo_issues") as repo_issues_mock: + updater = GithubUpdater(library=library) + repo_issues_mock.return_value = github_api_repo_issues_response + updater.update_issues() + + assert Issue.objects.count() == expected_count + ids = [issue.id for issue in github_api_repo_issues_response] + issues = Issue.objects.filter(library=library, github_id__in=ids) + assert issues.exists() + assert issues.count() == expected_count + + # Test the values of a sample Issue + gh_issue = github_api_repo_issues_response[0] + issue = issues.get(github_id=gh_issue.id) + assert issue.title == gh_issue.title + assert issue.number == gh_issue.number + if gh_issue.state == "open": + assert issue.is_open + else: + assert not issue.is_open + assert issue.data == gh_issue + + expected_closed = parse(gh_issue["closed_at"]) + expected_created = parse(gh_issue["created_at"]) + expected_modified = parse(gh_issue["updated_at"]) + assert issue.closed == expected_closed + assert issue.created == expected_created + assert issue.modified == expected_modified + + +def test_update_issues_existing(tp, library, github_api_repo_issues_response): + """Test that GithubUpdater.update_issues() updates existing issues when appropriate""" + existing_issue_data = github_api_repo_issues_response[0] + old_title = "Old title" + issue = baker.make( + Issue, library=library, github_id=existing_issue_data.id, title=old_title + ) + + # Make sure we are expected one fewer new issue, since we created one in advance + new_issues_count = len(github_api_repo_issues_response) + expected_count = Issue.objects.count() + new_issues_count - 1 + + with patch("libraries.github.repo_issues") as repo_issues_mock: + updater = GithubUpdater(library=library) + repo_issues_mock.return_value = github_api_repo_issues_response + updater.update_issues() + + assert Issue.objects.count() == expected_count + ids = [issue.id for issue in github_api_repo_issues_response] + issues = Issue.objects.filter(library=library, github_id__in=ids) + assert issues.exists() + assert issues.count() == expected_count + + # Test that the existing issue updated + issue.refresh_from_db() + assert issue.title == existing_issue_data.title + + +def test_update_issues_long_title(tp, library, github_api_repo_issues_response): + """Test that GithubUpdater.update_issues() handles long title gracefully""" + new_issues_count = len(github_api_repo_issues_response) + expected_count = Issue.objects.count() + new_issues_count + title = "sample" * 100 + assert len(title) > 255 + expected_title = title[:255] + assert len(expected_title) <= 255 + + with patch("libraries.github.repo_issues") as repo_issues_mock: + updater = GithubUpdater(library=library) + # Make an extra-long title so we can confirm that it saves + github_id = github_api_repo_issues_response[0]["id"] + github_api_repo_issues_response[0]["title"] = "sample" * 100 + repo_issues_mock.return_value = github_api_repo_issues_response + updater.update_issues() + + assert Issue.objects.count() == expected_count + assert Issue.objects.filter(library=library, github_id=github_id).exists() + issue = Issue.objects.get(library=library, github_id=github_id) + assert issue.title == expected_title + + +def test_update_prs_new(tp, library, github_api_repo_prs_response): + """Test that GithubUpdater.update_prs() imports new PRs appropriately""" + new_prs_count = len(github_api_repo_prs_response) + expected_count = PullRequest.objects.count() + new_prs_count + + with patch("libraries.github.repo_prs") as repo_prs_mock: + updater = GithubUpdater(library=library) + github_api_repo_prs_response[0]["title"] = "sample" * 100 + repo_prs_mock.return_value = github_api_repo_prs_response + updater.update_prs() + + assert PullRequest.objects.count() == expected_count + ids = [pr.id for pr in github_api_repo_prs_response] + pulls = PullRequest.objects.filter(library=library, github_id__in=ids) + assert pulls.exists() + assert pulls.count() == expected_count + + # Test the values of a sample PR + gh_pull = github_api_repo_prs_response[0] + pr = pulls.get(github_id=gh_pull.id) + assert pr.title == gh_pull.title[:255] + assert pr.number == gh_pull.number + if gh_pull.state == "open": + assert pr.is_open + else: + assert not pr.is_open + assert pr.data == gh_pull + + expected_closed = parse(gh_pull["closed_at"]) + expected_created = parse(gh_pull["created_at"]) + expected_modified = parse(gh_pull["updated_at"]) + assert pr.closed == expected_closed + assert pr.created == expected_created + assert pr.modified == expected_modified + + +def test_update_prs_existing(tp, library, github_api_repo_prs_response): + """Test that GithubUpdater.update_prs() updates existing PRs when appropriate""" + existing_pr_data = github_api_repo_prs_response[0] + old_title = "Old title" + pull = baker.make( + PullRequest, library=library, github_id=existing_pr_data.id, title=old_title + ) + + # Make sure we are expected one fewer new PRs, since we created one in advance + new_prs_count = len(github_api_repo_prs_response) + expected_count = PullRequest.objects.count() + new_prs_count - 1 + + with patch("libraries.github.repo_prs") as repo_prs_mock: + updater = GithubUpdater(library=library) + repo_prs_mock.return_value = github_api_repo_prs_response + updater.update_prs() + + assert PullRequest.objects.count() == expected_count + ids = [pr.id for pr in github_api_repo_prs_response] + pulls = PullRequest.objects.filter(library=library, github_id__in=ids) + assert pulls.exists() + assert pulls.count() == expected_count + + # Test that the existing PR updated + pull.refresh_from_db() + assert pull.title == existing_pr_data.title + + # LibraryUpdater tests diff --git a/libraries/tests/test_utils.py b/libraries/tests/test_utils.py new file mode 100644 index 00000000..3f8b91de --- /dev/null +++ b/libraries/tests/test_utils.py @@ -0,0 +1,22 @@ +from datetime import datetime + +from libraries.utils import parse_date + + +def test_parse_date_iso(): + expected = datetime.now() + result = parse_date(expected.isoformat()) + assert expected == result + + +def test_parse_date_str(): + expected = datetime.now() + input_date = f"{expected.month}-{expected.day}-{expected.year}" + result = parse_date(input_date) + assert expected.date() == result.date() + + +def test_parse_date_str_none(): + expected = None + result = parse_date("") + assert expected == result diff --git a/libraries/tests/test_views.py b/libraries/tests/test_views.py index 19b0a966..396f5ba1 100644 --- a/libraries/tests/test_views.py +++ b/libraries/tests/test_views.py @@ -1,15 +1,50 @@ -from unittest.mock import patch +from model_bakery import baker def test_library_list(library, tp): + """GET /libraries/""" res = tp.get("libraries") tp.response_200(res) def test_library_detail(library, tp): + """GET /libraries/{repo}/""" url = tp.reverse("library-detail", library.slug) + response = tp.get(url) + tp.response_200(response) - with patch("libraries.views.LibraryDetail.get_open_issues_count") as count_mock: - count_mock.return_value = 21 - res = tp.get(url) - tp.response_200(res) + +def test_library_detail_context_get_closed_prs_count(tp, library): + """ + GET /libraries/{repo}/ + Test that the custom closed_prs_count var appears as expected + """ + # Create open and closed PRs for this library, and another random PR + lib2 = baker.make("libraries.Library", slug="sample") + baker.make("libraries.PullRequest", library=library, is_open=True) + baker.make("libraries.PullRequest", library=library, is_open=False) + baker.make("libraries.PullRequest", library=lib2, is_open=True) + url = tp.reverse("library-detail", library.slug) + response = tp.get(url) + tp.response_200(response) + assert "closed_prs_count" in response.context + # Verify that the count only includes the one open PR for this library + assert response.context["closed_prs_count"] == 1 + + +def test_library_detail_context_get_open_issues_count(tp, library): + """ + GET /libraries/{repo}/ + Test that the custom open_issues_count var appears as expected + """ + # Create open and closed issues for this library, and another random issue + lib2 = baker.make("libraries.Library", slug="sample") + baker.make("libraries.Issue", library=library, is_open=True) + baker.make("libraries.Issue", library=library, is_open=False) + baker.make("libraries.Issue", library=lib2, is_open=True) + url = tp.reverse("library-detail", library.slug) + response = tp.get(url) + tp.response_200(response) + assert "open_issues_count" in response.context + # Verify that the count only includes the one open issue for this library + assert response.context["open_issues_count"] == 1 diff --git a/libraries/utils.py b/libraries/utils.py new file mode 100644 index 00000000..6ba1b1ef --- /dev/null +++ b/libraries/utils.py @@ -0,0 +1,14 @@ +import structlog + +from dateutil.parser import ParserError, parse + +logger = structlog.get_logger() + + +def parse_date(date_str): + """Parses a date string to a datetime. Does not return an error.""" + try: + return parse(date_str) + except ParserError: + logger.info("parse_date_invalid_date", date_str=date_str) + return None diff --git a/libraries/views.py b/libraries/views.py index bfa82fea..330eed67 100644 --- a/libraries/views.py +++ b/libraries/views.py @@ -1,7 +1,6 @@ from django.views.generic import DetailView, ListView -from .github import repo_issues -from .models import Category, Library +from .models import Category, Issue, Library, PullRequest class CategoryMixin: @@ -60,14 +59,12 @@ class LibraryDetail(CategoryMixin, DetailView): def get(self, request, *args, **kwargs): self.object = self.get_object() context = self.get_context_data(object=self.object) + context["closed_prs_count"] = self.get_closed_prs_count(self.object) context["open_issues_count"] = self.get_open_issues_count(self.object) return self.render_to_response(context) + def get_closed_prs_count(self, obj): + return PullRequest.objects.filter(library=obj, is_open=True).count() + def get_open_issues_count(self, obj): - try: - issues = repo_issues( - obj.github_owner, obj.github_repo, state="open", issues_only=True - ) - return len(issues) - except Exception: - return 0 + return Issue.objects.filter(library=obj, is_open=True).count() diff --git a/templates/libraries/detail.html b/templates/libraries/detail.html index 6a0bcc9e..a586eaa7 100644 --- a/templates/libraries/detail.html +++ b/templates/libraries/detail.html @@ -63,7 +63,7 @@

Closed Pull Requests

- X per Month + {{ closed_prs_count }}