- Add API endpoint to import versions

- Move import library versions logic into task
- Add API docs

config/urls.py

@@ -54,7 +54,7 @@ from users.views import (
     ProfileView,
     UserViewSet,
 )
-from versions.api import VersionViewSet
+from versions.api import ImportVersionsView, VersionViewSet
 from versions.views import VersionCurrentReleaseDetail, VersionDetail
 
 router = routers.SimpleRouter()
@@ -73,6 +73,11 @@ urlpatterns = (
         path("users/me/", CurrentUserProfileView.as_view(), name="profile-account"),
         path("users/<int:pk>/", ProfileView.as_view(), name="profile-user"),
         path("api/v1/users/me/", CurrentUserAPIView.as_view(), name="current-user"),
+        path(
+            "api/v1/import-versions/",
+            ImportVersionsView.as_view(),
+            name="import-versions",
+        ),
         path("api/v1/", include(router.urls)),
         path("200", OKView.as_view(), name="ok"),
         path("403", ForbiddenView.as_view(), name="forbidden"),

docs/README.md

@@ -5,3 +5,4 @@
 - [Management Commands](./commands.md)
 - [Retrieving Static Content from the Boost Amazon S3 Bucket](./static_content.md)
 - [Syncing Data about Boost Versions and Libraries with GitHub](./syncing_data_with_github.md)
+- [API Documentation](./api.md) - We don't have many API endpoints, but the ones we do have are documented here

docs/api.md

@@ -0,0 +1,10 @@
+# API Documentation
+
+## `/api/v1/import-versions/`
+
+- **Allowed methods:** POST only; no payload
+- **Payload**: None
+- **Permissions**: Limited to staff users
+- Imports all Boost releases that are not already in the database
+- Ignores beta releases, release candidates, etc.
+- Will also import library-versions, maintainers, and library-version documentation links

libraries/management/commands/import_library_versions.py

@@ -1,13 +1,9 @@
 import djclick as click
-import requests
 
 from django.conf import settings
 
-from libraries.github import LibraryUpdater
-from core.githubhelper import GithubAPIClient, GithubDataParser
-from libraries.models import Library, LibraryVersion
-from libraries.tasks import get_and_store_library_version_documentation_urls_for_version
 from versions.models import Version
+from versions.tasks import import_library_versions
 
 
 @click.command()
@@ -37,11 +33,6 @@ def command(min_release, release, token):
             1.7.1, 1.7.2, etc.)
     """
     click.secho("Saving library-version relationships...", fg="green")
-    client = GithubAPIClient(token=token)
-    parser = GithubDataParser()
-    updater = LibraryUpdater(client=client)
-
-    skipped = []
 
     min_release = f"boost-{min_release}"
     if release is None:
@@ -52,107 +43,7 @@ def command(min_release, release, token):
         )
 
     for version in versions.order_by("-name"):
-        ref = client.get_ref(ref=f"tags/{version.name}")
-        raw_gitmodules = client.get_gitmodules(ref=ref)
-        if not raw_gitmodules:
-            skipped.append(
-                {"version": version.name, "reason": "Invalid gitmodules file"}
-            )
-            continue
-
-        gitmodules = parser.parse_gitmodules(raw_gitmodules.decode("utf-8"))
-
-        for gitmodule in gitmodules:
-            reason = ""
-            library_name = gitmodule["module"]
-
-            click.echo(f"Processing module {library_name}...")
-
-            if library_name in updater.skip_modules:
-                continue
-
-            try:
-                libraries_json = client.get_libraries_json(
-                    repo_slug=library_name, tag=version.name
-                )
-            except requests.exceptions.JSONDecodeError:
-                reason = "libraries.json file was invalid"
-            except requests.exceptions.HTTPError:
-                reason = "libraries.json file not found"
-            except Exception as e:
-                reason = str(e)
-
-            if not libraries_json:
-                # Can happen with older releases
-                library_version = save_library_version_by_library_key(
-                    library_name, version, gitmodule
-                )
-                if library_version:
-                    if not reason:
-                        reason = "failure with libraries.json file"
-                    click.secho(f"{library_name} ({version.name} saved.", fg="green")
-                    continue
-                else:
-                    if not reason:
-                        reason = """
-                            Could not find libraries.json file, and could not find
-                            library by gitmodule name
-                        """
-                    skipped.append(
-                        {
-                            "version": version.name,
-                            "library": library_name,
-                            "reason": reason,
-                        }
-                    )
-                    continue
-
-            libraries = (
-                libraries_json if isinstance(libraries_json, list) else [libraries_json]
-            )
-            parsed_libraries = [parser.parse_libraries_json(lib) for lib in libraries]
-            for lib_data in parsed_libraries:
-                library, created = Library.objects.get_or_create(
-                    key=lib_data["key"],
-                    defaults={
-                        "name": lib_data.get("name"),
-                        "description": lib_data.get("description"),
-                        "cpp_standard_minimum": lib_data.get("cxxstd"),
-                        "data": lib_data,
-                    },
-                )
-                library_version, _ = LibraryVersion.objects.update_or_create(
-                    version=version, library=library, defaults={"data": lib_data}
-                )
-                click.secho(f"{library.name} ({version.name} saved)", fg="green")
-                # if created and not library.github_url:
-                if not library.github_url:
-                    pass
-                #     # todo: handle this. Need a github_url for these.
-
-        # Retrieve and store the docs url for each library-version in this release
-        get_and_store_library_version_documentation_urls_for_version.delay(version.pk)
-
-    skipped_messages = [
-        f"Skipped {obj['library']} in {obj['version']}: {obj['reason']}"
-        if "library" in obj
-        else f"Skipped {obj['version']}: {obj['reason']}"
-        for obj in skipped
-    ]
-
-    for message in skipped_messages:
-        click.secho(message, fg="red")
+        click.secho(f"Saving libraries for version {version.name}", fg="green")
+        import_library_versions.delay(version.name, token=token)
 
     click.secho("Finished saving library-version relationships.", fg="green")
-
-
-def save_library_version_by_library_key(library_key, version, gitmodule={}):
-    """Saves a LibraryVersion instance by library key and version."""
-    try:
-        library = Library.objects.get(key=library_key)
-        library_version, _ = LibraryVersion.objects.update_or_create(
-            version=version, library=library, defaults={"data": gitmodule}
-        )
-        return library_version
-    except Library.DoesNotExist:
-        return

versions/api.py

@@ -5,6 +5,23 @@ from versions.permissions import SuperUserOrVersionManager
 from versions.models import Version
 from versions.serializers import VersionSerializer
 
+from django.http import JsonResponse
+from django.views import View
+from django.contrib.admin.views.decorators import staff_member_required
+from django.utils.decorators import method_decorator
+from versions.tasks import import_versions
+
+
+@method_decorator(staff_member_required, name="dispatch")
+class ImportVersionsView(View):
+    """
+    API view to import versions, accessible only to staff and superusers.
+    """
+
+    def post(self, request, *args, **kwargs):
+        import_versions.delay(new_versions_only=True)
+        return JsonResponse({"status": "Importing versions..."}, status=200)
+
 
 class VersionViewSet(viewsets.ModelViewSet):
     model = Version

versions/tasks.py

@@ -1,3 +1,4 @@
+import requests
 import structlog
 
 from config.celery import app
@@ -5,6 +6,9 @@ from django.conf import settings
 from django.core.management import call_command
 from fastcore.xtras import obj2dict
 from core.githubhelper import GithubAPIClient, GithubDataParser
+from libraries.github import LibraryUpdater
+from libraries.models import Library, LibraryVersion
+from libraries.tasks import get_and_store_library_version_documentation_urls_for_version
 from versions.models import Version
 
 
@@ -94,14 +98,104 @@ def import_versions(delete_versions=False, new_versions_only=False, token=None):
         import_release_downloads.delay(version.pk)
 
         # Load library-versions
-        version_num = version.name.replace("boost-", "")
-        import_library_versions.delay(version_num)
+        import_library_versions.delay(version.name, token=token)
 
 
 @app.task
-def import_library_versions(version_num):
-    """version_num should be in the format N.NN.N, as in 1.83.0"""
-    call_command("import_library_versions", "--release", version_num)
+def import_library_versions(version_name, token=None):
+    """For a specific version, imports all LibraryVersions using GitHub data"""
+    try:
+        version = Version.objects.get(name=version_name)
+    except Version.DoesNotExist:
+        logger.info(
+            "import_library_versions_version_not_found", version_name=version_name
+        )
+
+    client = GithubAPIClient(token=token)
+    updater = LibraryUpdater(client=client)
+    parser = GithubDataParser()
+
+    # Get the gitmodules file for the version, which contains library data
+    ref = client.get_ref(ref=f"tags/{version_name}")
+    raw_gitmodules = client.get_gitmodules(ref=ref)
+    if not raw_gitmodules:
+        logger.info(
+            "import_library_versions_invalid_gitmodules", version_name=version_name
+        )
+        return
+
+    gitmodules = parser.parse_gitmodules(raw_gitmodules.decode("utf-8"))
+
+    # For each gitmodule, gets its libraries.json file and save the libraries
+    # to the version
+    for gitmodule in gitmodules:
+        library_name = gitmodule["module"]
+        if library_name in updater.skip_modules:
+            continue
+
+        try:
+            libraries_json = client.get_libraries_json(
+                repo_slug=library_name, tag=version_name
+            )
+        except (
+            requests.exceptions.JSONDecodeError,
+            requests.exceptions.HTTPError,
+            Exception,
+        ):
+            # Can happen with older releases
+            library_version = save_library_version_by_library_key(
+                library_name, version, gitmodule
+            )
+            if library_version:
+                logger.info(
+                    "import_library_versions_by_library_key",
+                    version_name=version_name,
+                    library_name=library_name,
+                )
+            else:
+                logger.info(
+                    "import_library_versions_skipped_library",
+                    version_name=version_name,
+                    library_name=library_name,
+                )
+            continue
+
+        if not libraries_json:
+            # Can happen with older releases -- we try to catch all exceptions
+            # so this is just in case
+            logger.info(
+                "import_library_versions_skipped_library",
+                version_name=version_name,
+                library_name=library_name,
+            )
+            continue
+
+        libraries = (
+            libraries_json if isinstance(libraries_json, list) else [libraries_json]
+        )
+        parsed_libraries = [parser.parse_libraries_json(lib) for lib in libraries]
+        for lib_data in parsed_libraries:
+            library, created = Library.objects.get_or_create(
+                key=lib_data["key"],
+                defaults={
+                    "name": lib_data.get("name"),
+                    "description": lib_data.get("description"),
+                    "cpp_standard_minimum": lib_data.get("cxxstd"),
+                    "data": lib_data,
+                },
+            )
+            library_version, _ = LibraryVersion.objects.update_or_create(
+                version=version, library=library, defaults={"data": lib_data}
+            )
+            if not library.github_url:
+                pass
+            #     # todo: handle this. Need a github_url for these.
+
+    # Retrieve and store the docs url for each library-version in this release
+    get_and_store_library_version_documentation_urls_for_version.delay(version.pk)
+
+    # Load maintainers for library-versions
+    call_command("update_maintainers", "--release", version.name)
 
 
 @app.task
@@ -158,3 +252,18 @@ def get_release_date_for_version(version_pk, commit_sha, token=None):
         logger.info("get_release_date_for_version_success", version_pk=version_pk)
     else:
         logger.error("get_release_date_for_version_error", version_pk=version_pk)
+
+
+# Helper functions
+
+
+def save_library_version_by_library_key(library_key, version, gitmodule={}):
+    """Saves a LibraryVersion instance by library key and version."""
+    try:
+        library = Library.objects.get(key=library_key)
+        library_version, _ = LibraryVersion.objects.update_or_create(
+            version=version, library=library, defaults={"data": gitmodule}
+        )
+        return library_version
+    except Library.DoesNotExist:
+        return

versions/tests/test_api.py

@@ -1,3 +1,27 @@
+from unittest.mock import patch
+from django.contrib.auth import get_user_model
+
+
+User = get_user_model()
+
+
 def test_public_view(full_version_one, tp):
     r = tp.client.get("/api/v1/versions/")
     tp.response_200(r)
+
+
+def test_import_versions_view(user, staff_user, tp):
+    """
+    POST /api/v1/import-versions/
+    """
+    with patch("versions.tasks.import_versions.delay") as mock_task, tp.login(
+        staff_user
+    ):
+        response = tp.post("import-versions")
+        mock_task.assert_called_once()
+        tp.response_200(response)
+
+    with patch("versions.tasks.import_versions.delay") as mock_task, tp.login(user):
+        response = tp.post("import-versions")
+        mock_task.assert_not_called()
+        tp.response_302(response)