From 79120e12cb9722ea0e109c9594d69747cb16e1cc Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 16 Dec 2022 10:13:53 -0800 Subject: [PATCH] :construction: Progress on PR import --- libraries/github.py | 109 ++++++++++- libraries/tests/fixtures.py | 369 ++++++++++++++++++++++++++++++++++++ 2 files changed, 475 insertions(+), 3 deletions(-) diff --git a/libraries/github.py b/libraries/github.py index 304aa525..a62441a6 100644 --- a/libraries/github.py +++ b/libraries/github.py @@ -7,7 +7,9 @@ import structlog from dateutil.parser import ParserError, parse from ghapi.all import GhApi, paged -from .models import Category, Library + +from .models import Category, Issue, Library, PullRequest +from .utils import parse_date logger = structlog.get_logger() @@ -321,8 +323,109 @@ class GithubUpdater: def update_issues(self): self.logger.info("updating_repo_issues") - issues = repo_issues(self.owner, self.repo, state="all") + + issues_data = repo_issues( + self.owner, self.library.name, state="all", issues_only=True + ) + + for issue_dict in issues_data: + + # Get the date information + closed_at = None + created_at = None + modified_at = None + + if issue_dict.get("closed_at"): + closed_at = parse_date(issue_dict["closed_at"]) + + if issue_dict.get("created_at"): + created_at = parse_date(issue_dict["created_at"]) + + if issue_dict.get("updated_at"): + modified_at = parse_date(issue_dict["updated_at"]) + + # Create or update the Issue object + try: + issue, created = Issue.objects.update_or_create( + library=self.library, + github_id=issue_dict["id"], + defaults={ + "title": issue_dict["title"][:255], + "number": issue_dict["number"], + "is_open": issue_dict["state"] == "open", + "closed": closed_at, + "created": created_at, + "modified": modified_at, + "data": obj2dict(issue_dict), + }, + ) + except Exception as e: + logger.exception( + "update_issues_error_skipped_issue", + issue_github_id=issue_dict.get("id"), + exc_msg=str(e), + ) + continue + logger.info( + "issue_updated_successfully", + issue_id=issue.id, + created=created, + issue_github_id=issue.github_id, + ) def update_prs(self): + """Update all PRs for a library""" self.logger.info("updating_repo_prs") - raise ValueError("testing!") + + prs_data = repo_prs(self.owner, self.library.name) + + for pr_dict in prs_data: + # Get the date information + closed_at = None + merged_at = None + created_at = None + modified_at = None + + if pr_dict.get("closed_at"): + closed_at = parse_date(pr_dict["closed_at"]) + + if pr_dict.get("merged_at"): + merged_at = parse_date(pr_dict["merged_at"]) + + if pr_dict.get("created_at"): + created_at = parse_date(pr_dict["created_at"]) + + if pr_dict.get("modified_at"): + modified_at = parse_date(pr_dict["modified_at"]) + + breakpoint() + + try: + pull_request, created = PullRequest.objects.update_or_create( + library=self.library, + github_id=pr_dict["id"], + defaults={ + "title": pr_dict["title"][:255], + "number": pr_dict["number"], + "is_open": pr_dict["state"] == "open", + "closed": closed_at, + "merged": merged_at, + "created": created_at, + "modified": modified_at, + "data": obj2dict(pr_dict), + }, + ) + except Exception as e: + logger.exception( + "update_prs_error_skipped_pr", + pr_github_id=pr_dict.get("id"), + exc_msg=str(e), + ) + continue + + logger.info( + "pr_updated_successfully", + pull_request_id=pull_request.id, + created=created, + pull_request_github_id=pull_request.github_id, + ) diff --git a/libraries/tests/fixtures.py b/libraries/tests/fixtures.py index b6661541..259a3000 100644 --- a/libraries/tests/fixtures.py +++ b/libraries/tests/fixtures.py @@ -60,6 +60,375 @@ def github_api_get_repo_response(db): return {"updated_at": "2022-09-14T22:20:38Z"} +@pytest.fixture +def github_api_repo_issues_response(db): + """Returns the response from GhApi().issues.list_for_repo, already paged""" + return [ + dict2obj( + { + "title": "Issue Number One", + "number": 1, + "state": "closed", + "closed_at": "2022-04-11T12:38:24Z", + "created_at": "2022-04-11T11:41:02Z", + "updated_at": "2022-04-11T12:38:25Z", + "id": 5898798798, + } + ), + dict2obj( + { + "title": "Issue Number Two", + "number": 2, + "state": "open", + "closed_at": "2022-04-11T12:38:24Z", + "created_at": "2022-04-11T11:41:02Z", + "updated_at": "2022-04-11T12:38:25Z", + "id": 7395968281, + } + ), + dict2obj( + { + "title": "Issue Number Three", + "number": 3, + "state": "closed", + "closed_at": "2022-04-11T12:38:24Z", + "created_at": "2022-04-11T11:41:02Z", + "updated_at": "2022-04-11T12:38:25Z", + "id": 7492027464, + } + ), + ] + + +@pytest.fixture +def github_api_repo_prs_response(db): + """Returns the response from GhApi().pulls.list, already paged""" + return { + "url": "https://api.github.com/repos/boostorg/system/pulls/90", + "id": 1032140532, + "node_id": "PR_kwDOAHPQi849hTb0", + "html_url": "https://github.com/boostorg/system/pull/90", + "diff_url": "https://github.com/boostorg/system/pull/90.diff", + "patch_url": "https://github.com/boostorg/system/pull/90.patch", + "issue_url": "https://api.github.com/repos/boostorg/system/issues/90", + "number": 90, + "state": "open", + "locked": False, + "title": "add boost_system.natvis and interface source files", + "user": { + "login": "vinniefalco", + "id": 1503976, + "node_id": "MDQ6VXNlcjE1MDM5NzY=", + "avatar_url": "https://avatars.githubusercontent.com/u/1503976?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/vinniefalco", + "html_url": "https://github.com/vinniefalco", + "followers_url": "https://api.github.com/users/vinniefalco/followers", + "following_url": "https://api.github.com/users/vinniefalco/following{/other_user}", + "gists_url": "https://api.github.com/users/vinniefalco/gists{/gist_id}", + "starred_url": "https://api.github.com/users/vinniefalco/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/vinniefalco/subscriptions", + "organizations_url": "https://api.github.com/users/vinniefalco/orgs", + "repos_url": "https://api.github.com/users/vinniefalco/repos", + "events_url": "https://api.github.com/users/vinniefalco/events{/privacy}", + "received_events_url": "https://api.github.com/users/vinniefalco/received_events", + "type": "User", + "site_admin": False, + }, + "body": None, + "created_at": "2022-08-21T22:24:43Z", + "updated_at": "2022-08-21T22:24:43Z", + "closed_at": None, + "merged_at": None, + "merge_commit_sha": "37653ac206475a046d7e7abadaf823430e564572", + "assignee": None, + "assignees": [], + "requested_reviewers": [], + "requested_teams": [], + "labels": [], + "milestone": None, + "draft": False, + "commits_url": "https://api.github.com/repos/boostorg/system/pulls/90/commits", + "review_comments_url": "https://api.github.com/repos/boostorg/system/pulls/90/comments", + "review_comment_url": "https://api.github.com/repos/boostorg/system/pulls/comments{/number}", + "comments_url": "https://api.github.com/repos/boostorg/system/issues/90/comments", + "statuses_url": "https://api.github.com/repos/boostorg/system/statuses/fe48c3058daaa31da6c50c316d63aa5f185dacb8", + "head": { + "label": "vinniefalco:natvis", + "ref": "natvis", + "sha": "fe48c3058daaa31da6c50c316d63aa5f185dacb8", + "user": { + "login": "vinniefalco", + "id": 1503976, + "node_id": "MDQ6VXNlcjE1MDM5NzY=", + "avatar_url": "https://avatars.githubusercontent.com/u/1503976?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/vinniefalco", + "html_url": "https://github.com/vinniefalco", + "followers_url": "https://api.github.com/users/vinniefalco/followers", + "following_url": "https://api.github.com/users/vinniefalco/following{/other_user}", + "gists_url": "https://api.github.com/users/vinniefalco/gists{/gist_id}", + "starred_url": "https://api.github.com/users/vinniefalco/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/vinniefalco/subscriptions", + "organizations_url": "https://api.github.com/users/vinniefalco/orgs", + "repos_url": "https://api.github.com/users/vinniefalco/repos", + "events_url": "https://api.github.com/users/vinniefalco/events{/privacy}", + "received_events_url": "https://api.github.com/users/vinniefalco/received_events", + "type": "User", + "site_admin": False, + }, + "repo": { + "id": 526406204, + "node_id": "R_kgDOH2BSPA", + "name": "boost-system", + "full_name": "vinniefalco/boost-system", + "private": False, + "owner": { + "login": "vinniefalco", + "id": 1503976, + "node_id": "MDQ6VXNlcjE1MDM5NzY=", + "avatar_url": "https://avatars.githubusercontent.com/u/1503976?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/vinniefalco", + "html_url": "https://github.com/vinniefalco", + "followers_url": "https://api.github.com/users/vinniefalco/followers", + "following_url": "https://api.github.com/users/vinniefalco/following{/other_user}", + "gists_url": "https://api.github.com/users/vinniefalco/gists{/gist_id}", + "starred_url": "https://api.github.com/users/vinniefalco/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/vinniefalco/subscriptions", + "organizations_url": "https://api.github.com/users/vinniefalco/orgs", + "repos_url": "https://api.github.com/users/vinniefalco/repos", + "events_url": "https://api.github.com/users/vinniefalco/events{/privacy}", + "received_events_url": "https://api.github.com/users/vinniefalco/received_events", + "type": "User", + "site_admin": False, + }, + "html_url": "https://github.com/vinniefalco/boost-system", + "description": "Boost.org system module ", + "fork": True, + "url": "https://api.github.com/repos/vinniefalco/boost-system", + "forks_url": "https://api.github.com/repos/vinniefalco/boost-system/forks", + "keys_url": "https://api.github.com/repos/vinniefalco/boost-system/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/vinniefalco/boost-system/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/vinniefalco/boost-system/teams", + "hooks_url": "https://api.github.com/repos/vinniefalco/boost-system/hooks", + "issue_events_url": "https://api.github.com/repos/vinniefalco/boost-system/issues/events{/number}", + "events_url": "https://api.github.com/repos/vinniefalco/boost-system/events", + "assignees_url": "https://api.github.com/repos/vinniefalco/boost-system/assignees{/user}", + "branches_url": "https://api.github.com/repos/vinniefalco/boost-system/branches{/branch}", + "tags_url": "https://api.github.com/repos/vinniefalco/boost-system/tags", + "blobs_url": "https://api.github.com/repos/vinniefalco/boost-system/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/vinniefalco/boost-system/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/vinniefalco/boost-system/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/vinniefalco/boost-system/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/vinniefalco/boost-system/statuses/{sha}", + "languages_url": "https://api.github.com/repos/vinniefalco/boost-system/languages", + "stargazers_url": "https://api.github.com/repos/vinniefalco/boost-system/stargazers", + "contributors_url": "https://api.github.com/repos/vinniefalco/boost-system/contributors", + "subscribers_url": "https://api.github.com/repos/vinniefalco/boost-system/subscribers", + "subscription_url": "https://api.github.com/repos/vinniefalco/boost-system/subscription", + "commits_url": "https://api.github.com/repos/vinniefalco/boost-system/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/vinniefalco/boost-system/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/vinniefalco/boost-system/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/vinniefalco/boost-system/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/vinniefalco/boost-system/contents/{+path}", + "compare_url": "https://api.github.com/repos/vinniefalco/boost-system/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/vinniefalco/boost-system/merges", + "archive_url": "https://api.github.com/repos/vinniefalco/boost-system/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/vinniefalco/boost-system/downloads", + "issues_url": "https://api.github.com/repos/vinniefalco/boost-system/issues{/number}", + "pulls_url": "https://api.github.com/repos/vinniefalco/boost-system/pulls{/number}", + "milestones_url": "https://api.github.com/repos/vinniefalco/boost-system/milestones{/number}", + "notifications_url": "https://api.github.com/repos/vinniefalco/boost-system/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/vinniefalco/boost-system/labels{/name}", + "releases_url": "https://api.github.com/repos/vinniefalco/boost-system/releases{/id}", + "deployments_url": "https://api.github.com/repos/vinniefalco/boost-system/deployments", + "created_at": "2022-08-18T23:48:50Z", + "updated_at": "2022-08-19T01:05:39Z", + "pushed_at": "2022-08-21T22:22:12Z", + "git_url": "git://github.com/vinniefalco/boost-system.git", + "ssh_url": "git@github.com:vinniefalco/boost-system.git", + "clone_url": "https://github.com/vinniefalco/boost-system.git", + "svn_url": "https://github.com/vinniefalco/boost-system", + "homepage": "http://boost.org/libs/system", + "size": 781, + "stargazers_count": 0, + "watchers_count": 0, + "language": "C++", + "has_issues": False, + "has_projects": True, + "has_downloads": True, + "has_wiki": True, + "has_pages": False, + "has_discussions": False, + "forks_count": 0, + "mirror_url": None, + "archived": False, + "disabled": False, + "open_issues_count": 0, + "license": None, + "allow_forking": True, + "is_template": False, + "web_commit_signoff_required": False, + "topics": [], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "develop", + }, + }, + "base": { + "label": "boostorg:develop", + "ref": "develop", + "sha": "8c740705e6a221ef5fed7402338ba475df84077d", + "user": { + "login": "boostorg", + "id": 3170529, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjMxNzA1Mjk=", + "avatar_url": "https://avatars.githubusercontent.com/u/3170529?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/boostorg", + "html_url": "https://github.com/boostorg", + "followers_url": "https://api.github.com/users/boostorg/followers", + "following_url": "https://api.github.com/users/boostorg/following{/other_user}", + "gists_url": "https://api.github.com/users/boostorg/gists{/gist_id}", + "starred_url": "https://api.github.com/users/boostorg/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/boostorg/subscriptions", + "organizations_url": "https://api.github.com/users/boostorg/orgs", + "repos_url": "https://api.github.com/users/boostorg/repos", + "events_url": "https://api.github.com/users/boostorg/events{/privacy}", + "received_events_url": "https://api.github.com/users/boostorg/received_events", + "type": "Organization", + "site_admin": False, + }, + "repo": { + "id": 7590027, + "node_id": "MDEwOlJlcG9zaXRvcnk3NTkwMDI3", + "name": "system", + "full_name": "boostorg/system", + "private": False, + "owner": { + "login": "boostorg", + "id": 3170529, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjMxNzA1Mjk=", + "avatar_url": "https://avatars.githubusercontent.com/u/3170529?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/boostorg", + "html_url": "https://github.com/boostorg", + "followers_url": "https://api.github.com/users/boostorg/followers", + "following_url": "https://api.github.com/users/boostorg/following{/other_user}", + "gists_url": "https://api.github.com/users/boostorg/gists{/gist_id}", + "starred_url": "https://api.github.com/users/boostorg/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/boostorg/subscriptions", + "organizations_url": "https://api.github.com/users/boostorg/orgs", + "repos_url": "https://api.github.com/users/boostorg/repos", + "events_url": "https://api.github.com/users/boostorg/events{/privacy}", + "received_events_url": "https://api.github.com/users/boostorg/received_events", + "type": "Organization", + "site_admin": False, + }, + "html_url": "https://github.com/boostorg/system", + "description": "Boost.org system module ", + "fork": False, + "url": "https://api.github.com/repos/boostorg/system", + "forks_url": "https://api.github.com/repos/boostorg/system/forks", + "keys_url": "https://api.github.com/repos/boostorg/system/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/boostorg/system/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/boostorg/system/teams", + "hooks_url": "https://api.github.com/repos/boostorg/system/hooks", + "issue_events_url": "https://api.github.com/repos/boostorg/system/issues/events{/number}", + "events_url": "https://api.github.com/repos/boostorg/system/events", + "assignees_url": "https://api.github.com/repos/boostorg/system/assignees{/user}", + "branches_url": "https://api.github.com/repos/boostorg/system/branches{/branch}", + "tags_url": "https://api.github.com/repos/boostorg/system/tags", + "blobs_url": "https://api.github.com/repos/boostorg/system/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/boostorg/system/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/boostorg/system/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/boostorg/system/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/boostorg/system/statuses/{sha}", + "languages_url": "https://api.github.com/repos/boostorg/system/languages", + "stargazers_url": "https://api.github.com/repos/boostorg/system/stargazers", + "contributors_url": "https://api.github.com/repos/boostorg/system/contributors", + "subscribers_url": "https://api.github.com/repos/boostorg/system/subscribers", + "subscription_url": "https://api.github.com/repos/boostorg/system/subscription", + "commits_url": "https://api.github.com/repos/boostorg/system/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/boostorg/system/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/boostorg/system/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/boostorg/system/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/boostorg/system/contents/{+path}", + "compare_url": "https://api.github.com/repos/boostorg/system/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/boostorg/system/merges", + "archive_url": "https://api.github.com/repos/boostorg/system/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/boostorg/system/downloads", + "issues_url": "https://api.github.com/repos/boostorg/system/issues{/number}", + "pulls_url": "https://api.github.com/repos/boostorg/system/pulls{/number}", + "milestones_url": "https://api.github.com/repos/boostorg/system/milestones{/number}", + "notifications_url": "https://api.github.com/repos/boostorg/system/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/boostorg/system/labels{/name}", + "releases_url": "https://api.github.com/repos/boostorg/system/releases{/id}", + "deployments_url": "https://api.github.com/repos/boostorg/system/deployments", + "created_at": "2013-01-13T15:59:31Z", + "updated_at": "2022-12-14T22:25:46Z", + "pushed_at": "2022-12-14T15:17:31Z", + "git_url": "git://github.com/boostorg/system.git", + "ssh_url": "git@github.com:boostorg/system.git", + "clone_url": "https://github.com/boostorg/system.git", + "svn_url": "https://github.com/boostorg/system", + "homepage": "http://boost.org/libs/system", + "size": 852, + "stargazers_count": 26, + "watchers_count": 26, + "language": "C++", + "has_issues": True, + "has_projects": False, + "has_downloads": True, + "has_wiki": False, + "has_pages": False, + "has_discussions": False, + "forks_count": 82, + "mirror_url": None, + "archived": False, + "disabled": False, + "open_issues_count": 10, + "license": None, + "allow_forking": True, + "is_template": False, + "web_commit_signoff_required": False, + "topics": [], + "visibility": "public", + "forks": 82, + "open_issues": 10, + "watchers": 26, + "default_branch": "develop", + }, + }, + "_links": { + "self": {"href": "https://api.github.com/repos/boostorg/system/pulls/90"}, + "html": {"href": "https://github.com/boostorg/system/pull/90"}, + "issue": {"href": "https://api.github.com/repos/boostorg/system/issues/90"}, + "comments": { + "href": "https://api.github.com/repos/boostorg/system/issues/90/comments" + }, + "review_comments": { + "href": "https://api.github.com/repos/boostorg/system/pulls/90/comments" + }, + "review_comment": { + "href": "https://api.github.com/repos/boostorg/system/pulls/comments{/number}" + }, + "commits": { + "href": "https://api.github.com/repos/boostorg/system/pulls/90/commits" + }, + "statuses": { + "href": "https://api.github.com/repos/boostorg/system/statuses/fe48c3058daaa31da6c50c316d63aa5f185dacb8" + }, + }, + "author_association": "MEMBER", + "auto_merge": None, + "active_lock_reason": None, + } + + @pytest.fixture def boost_module(): return {"module": "rational", "url": "rational"}