diff --git a/config/urls.py b/config/urls.py index 08b742b8..aa879034 100755 --- a/config/urls.py +++ b/config/urls.py @@ -54,7 +54,7 @@ from users.views import ( ProfileView, UserViewSet, ) -from versions.api import VersionViewSet +from versions.api import ImportVersionsView, VersionViewSet from versions.views import VersionCurrentReleaseDetail, VersionDetail router = routers.SimpleRouter() @@ -73,6 +73,11 @@ urlpatterns = ( path("users/me/", CurrentUserProfileView.as_view(), name="profile-account"), path("users//", ProfileView.as_view(), name="profile-user"), path("api/v1/users/me/", CurrentUserAPIView.as_view(), name="current-user"), + path( + "api/v1/import-versions/", + ImportVersionsView.as_view(), + name="import-versions", + ), path("api/v1/", include(router.urls)), path("200", OKView.as_view(), name="ok"), path("403", ForbiddenView.as_view(), name="forbidden"), diff --git a/docs/README.md b/docs/README.md index 58582a66..960d83db 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,3 +5,4 @@ - [Management Commands](./commands.md) - [Retrieving Static Content from the Boost Amazon S3 Bucket](./static_content.md) - [Syncing Data about Boost Versions and Libraries with GitHub](./syncing_data_with_github.md) +- [API Documentation](./api.md) - We don't have many API endpoints, but the ones we do have are documented here diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 00000000..1c5b0c91 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,10 @@ +# API Documentation + +## `/api/v1/import-versions/` + +- **Allowed methods:** POST only; no payload +- **Payload**: None +- **Permissions**: Limited to staff users +- Imports all Boost releases that are not already in the database +- Ignores beta releases, release candidates, etc. +- Will also import library-versions, maintainers, and library-version documentation links diff --git a/libraries/management/commands/import_library_versions.py b/libraries/management/commands/import_library_versions.py index 413178ab..a4cdcd48 100644 --- a/libraries/management/commands/import_library_versions.py +++ b/libraries/management/commands/import_library_versions.py @@ -1,13 +1,9 @@ import djclick as click -import requests from django.conf import settings -from libraries.github import LibraryUpdater -from core.githubhelper import GithubAPIClient, GithubDataParser -from libraries.models import Library, LibraryVersion -from libraries.tasks import get_and_store_library_version_documentation_urls_for_version from versions.models import Version +from versions.tasks import import_library_versions @click.command() @@ -37,11 +33,6 @@ def command(min_release, release, token): 1.7.1, 1.7.2, etc.) """ click.secho("Saving library-version relationships...", fg="green") - client = GithubAPIClient(token=token) - parser = GithubDataParser() - updater = LibraryUpdater(client=client) - - skipped = [] min_release = f"boost-{min_release}" if release is None: @@ -52,107 +43,7 @@ def command(min_release, release, token): ) for version in versions.order_by("-name"): - ref = client.get_ref(ref=f"tags/{version.name}") - raw_gitmodules = client.get_gitmodules(ref=ref) - if not raw_gitmodules: - skipped.append( - {"version": version.name, "reason": "Invalid gitmodules file"} - ) - continue - - gitmodules = parser.parse_gitmodules(raw_gitmodules.decode("utf-8")) - - for gitmodule in gitmodules: - reason = "" - library_name = gitmodule["module"] - - click.echo(f"Processing module {library_name}...") - - if library_name in updater.skip_modules: - continue - - try: - libraries_json = client.get_libraries_json( - repo_slug=library_name, tag=version.name - ) - except requests.exceptions.JSONDecodeError: - reason = "libraries.json file was invalid" - except requests.exceptions.HTTPError: - reason = "libraries.json file not found" - except Exception as e: - reason = str(e) - - if not libraries_json: - # Can happen with older releases - library_version = save_library_version_by_library_key( - library_name, version, gitmodule - ) - if library_version: - if not reason: - reason = "failure with libraries.json file" - click.secho(f"{library_name} ({version.name} saved.", fg="green") - continue - else: - if not reason: - reason = """ - Could not find libraries.json file, and could not find - library by gitmodule name - """ - skipped.append( - { - "version": version.name, - "library": library_name, - "reason": reason, - } - ) - continue - - libraries = ( - libraries_json if isinstance(libraries_json, list) else [libraries_json] - ) - parsed_libraries = [parser.parse_libraries_json(lib) for lib in libraries] - for lib_data in parsed_libraries: - library, created = Library.objects.get_or_create( - key=lib_data["key"], - defaults={ - "name": lib_data.get("name"), - "description": lib_data.get("description"), - "cpp_standard_minimum": lib_data.get("cxxstd"), - "data": lib_data, - }, - ) - library_version, _ = LibraryVersion.objects.update_or_create( - version=version, library=library, defaults={"data": lib_data} - ) - click.secho(f"{library.name} ({version.name} saved)", fg="green") - # if created and not library.github_url: - if not library.github_url: - pass - # # todo: handle this. Need a github_url for these. - - # Retrieve and store the docs url for each library-version in this release - get_and_store_library_version_documentation_urls_for_version.delay(version.pk) - - skipped_messages = [ - f"Skipped {obj['library']} in {obj['version']}: {obj['reason']}" - if "library" in obj - else f"Skipped {obj['version']}: {obj['reason']}" - for obj in skipped - ] - - for message in skipped_messages: - click.secho(message, fg="red") + click.secho(f"Saving libraries for version {version.name}", fg="green") + import_library_versions.delay(version.name, token=token) click.secho("Finished saving library-version relationships.", fg="green") - - -def save_library_version_by_library_key(library_key, version, gitmodule={}): - """Saves a LibraryVersion instance by library key and version.""" - try: - library = Library.objects.get(key=library_key) - library_version, _ = LibraryVersion.objects.update_or_create( - version=version, library=library, defaults={"data": gitmodule} - ) - return library_version - except Library.DoesNotExist: - return diff --git a/versions/api.py b/versions/api.py index f6560fe4..b335a286 100755 --- a/versions/api.py +++ b/versions/api.py @@ -5,6 +5,23 @@ from versions.permissions import SuperUserOrVersionManager from versions.models import Version from versions.serializers import VersionSerializer +from django.http import JsonResponse +from django.views import View +from django.contrib.admin.views.decorators import staff_member_required +from django.utils.decorators import method_decorator +from versions.tasks import import_versions + + +@method_decorator(staff_member_required, name="dispatch") +class ImportVersionsView(View): + """ + API view to import versions, accessible only to staff and superusers. + """ + + def post(self, request, *args, **kwargs): + import_versions.delay(new_versions_only=True) + return JsonResponse({"status": "Importing versions..."}, status=200) + class VersionViewSet(viewsets.ModelViewSet): model = Version diff --git a/versions/tasks.py b/versions/tasks.py index 462b75ad..36dc7049 100644 --- a/versions/tasks.py +++ b/versions/tasks.py @@ -1,3 +1,4 @@ +import requests import structlog from config.celery import app @@ -5,6 +6,9 @@ from django.conf import settings from django.core.management import call_command from fastcore.xtras import obj2dict from core.githubhelper import GithubAPIClient, GithubDataParser +from libraries.github import LibraryUpdater +from libraries.models import Library, LibraryVersion +from libraries.tasks import get_and_store_library_version_documentation_urls_for_version from versions.models import Version @@ -94,14 +98,104 @@ def import_versions(delete_versions=False, new_versions_only=False, token=None): import_release_downloads.delay(version.pk) # Load library-versions - version_num = version.name.replace("boost-", "") - import_library_versions.delay(version_num) + import_library_versions.delay(version.name, token=token) @app.task -def import_library_versions(version_num): - """version_num should be in the format N.NN.N, as in 1.83.0""" - call_command("import_library_versions", "--release", version_num) +def import_library_versions(version_name, token=None): + """For a specific version, imports all LibraryVersions using GitHub data""" + try: + version = Version.objects.get(name=version_name) + except Version.DoesNotExist: + logger.info( + "import_library_versions_version_not_found", version_name=version_name + ) + + client = GithubAPIClient(token=token) + updater = LibraryUpdater(client=client) + parser = GithubDataParser() + + # Get the gitmodules file for the version, which contains library data + ref = client.get_ref(ref=f"tags/{version_name}") + raw_gitmodules = client.get_gitmodules(ref=ref) + if not raw_gitmodules: + logger.info( + "import_library_versions_invalid_gitmodules", version_name=version_name + ) + return + + gitmodules = parser.parse_gitmodules(raw_gitmodules.decode("utf-8")) + + # For each gitmodule, gets its libraries.json file and save the libraries + # to the version + for gitmodule in gitmodules: + library_name = gitmodule["module"] + if library_name in updater.skip_modules: + continue + + try: + libraries_json = client.get_libraries_json( + repo_slug=library_name, tag=version_name + ) + except ( + requests.exceptions.JSONDecodeError, + requests.exceptions.HTTPError, + Exception, + ): + # Can happen with older releases + library_version = save_library_version_by_library_key( + library_name, version, gitmodule + ) + if library_version: + logger.info( + "import_library_versions_by_library_key", + version_name=version_name, + library_name=library_name, + ) + else: + logger.info( + "import_library_versions_skipped_library", + version_name=version_name, + library_name=library_name, + ) + continue + + if not libraries_json: + # Can happen with older releases -- we try to catch all exceptions + # so this is just in case + logger.info( + "import_library_versions_skipped_library", + version_name=version_name, + library_name=library_name, + ) + continue + + libraries = ( + libraries_json if isinstance(libraries_json, list) else [libraries_json] + ) + parsed_libraries = [parser.parse_libraries_json(lib) for lib in libraries] + for lib_data in parsed_libraries: + library, created = Library.objects.get_or_create( + key=lib_data["key"], + defaults={ + "name": lib_data.get("name"), + "description": lib_data.get("description"), + "cpp_standard_minimum": lib_data.get("cxxstd"), + "data": lib_data, + }, + ) + library_version, _ = LibraryVersion.objects.update_or_create( + version=version, library=library, defaults={"data": lib_data} + ) + if not library.github_url: + pass + # # todo: handle this. Need a github_url for these. + + # Retrieve and store the docs url for each library-version in this release + get_and_store_library_version_documentation_urls_for_version.delay(version.pk) + + # Load maintainers for library-versions + call_command("update_maintainers", "--release", version.name) @app.task @@ -158,3 +252,18 @@ def get_release_date_for_version(version_pk, commit_sha, token=None): logger.info("get_release_date_for_version_success", version_pk=version_pk) else: logger.error("get_release_date_for_version_error", version_pk=version_pk) + + +# Helper functions + + +def save_library_version_by_library_key(library_key, version, gitmodule={}): + """Saves a LibraryVersion instance by library key and version.""" + try: + library = Library.objects.get(key=library_key) + library_version, _ = LibraryVersion.objects.update_or_create( + version=version, library=library, defaults={"data": gitmodule} + ) + return library_version + except Library.DoesNotExist: + return diff --git a/versions/tests/test_api.py b/versions/tests/test_api.py index 14a8e1d5..89772dd5 100644 --- a/versions/tests/test_api.py +++ b/versions/tests/test_api.py @@ -1,3 +1,27 @@ +from unittest.mock import patch +from django.contrib.auth import get_user_model + + +User = get_user_model() + + def test_public_view(full_version_one, tp): r = tp.client.get("/api/v1/versions/") tp.response_200(r) + + +def test_import_versions_view(user, staff_user, tp): + """ + POST /api/v1/import-versions/ + """ + with patch("versions.tasks.import_versions.delay") as mock_task, tp.login( + staff_user + ): + response = tp.post("import-versions") + mock_task.assert_called_once() + tp.response_200(response) + + with patch("versions.tasks.import_versions.delay") as mock_task, tp.login(user): + response = tp.post("import-versions") + mock_task.assert_not_called() + tp.response_302(response)