From e5268ed7d84e27b2e303db8afe8dd2ad9907a356 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Wed, 14 Feb 2024 13:01:29 -0800 Subject: [PATCH] Fix Container Hash and other docs: - Add LibraryVersion inline to LibraryAdmin for easier debugging - Add convenience function to generate random string - Adjust Library slug so it functions as unique without being unique - Add exception logic for older Container Hash versions - Fix bugs in management commands - Add exception logic for older Container Hash docs - Add exception logic for Functional/Overloaded Function docs - Add task to import all library-versions, and add that to existing admin refresh buttons - Add logic to import github_url if we dont have it --- libraries/admin.py | 13 +++++++++- .../import_library_version_docs_urls.py | 5 +--- .../commands/import_library_versions.py | 7 +++--- libraries/models.py | 24 ++++++++++++++++--- libraries/tasks.py | 21 +++++++++++++++- libraries/utils.py | 8 +++++++ versions/tasks.py | 14 +++++++++-- 7 files changed, 77 insertions(+), 15 deletions(-) diff --git a/libraries/admin.py b/libraries/admin.py index 2988db94..a15a8dad 100644 --- a/libraries/admin.py +++ b/libraries/admin.py @@ -3,6 +3,7 @@ from django.http import HttpResponseRedirect from django.urls import path, reverse from django.utils.html import format_html +from versions.tasks import import_all_library_versions from .models import Category, CommitData, Issue, Library, LibraryVersion, PullRequest from .tasks import ( update_commit_counts, @@ -83,6 +84,13 @@ class CommitDataAdmin(admin.ModelAdmin): return HttpResponseRedirect("../") +class LibraryVersionInline(admin.TabularInline): + model = LibraryVersion + extra = 0 + ordering = ["-version__name"] + fields = ["version", "documentation_url"] + + @admin.register(Library) class LibraryAdmin(admin.ModelAdmin): list_display = ["name", "key", "github_url"] @@ -90,6 +98,7 @@ class LibraryAdmin(admin.ModelAdmin): list_filter = ["categories"] ordering = ["name"] change_list_template = "admin/library_change_list.html" + inlines = [LibraryVersionInline] def get_urls(self): urls = super().get_urls() @@ -101,6 +110,7 @@ class LibraryAdmin(admin.ModelAdmin): def update_libraries(self, request): """Run the task to refresh the library data from GitHub""" update_libraries.delay() + import_all_library_versions.delay() self.message_user( request, """ @@ -126,7 +136,8 @@ class LibraryVersionAdmin(admin.ModelAdmin): return my_urls + urls def update_docs_urls(self, request): - """Run the task to refresh the documentation URLS from S3""" + """Run the task to refresh the documentation URLS from S3 and refresh data""" + import_all_library_versions.delay() update_library_version_documentation_urls_all_versions.delay() self.message_user( request, diff --git a/libraries/management/commands/import_library_version_docs_urls.py b/libraries/management/commands/import_library_version_docs_urls.py index 0bf1559e..cc869831 100644 --- a/libraries/management/commands/import_library_version_docs_urls.py +++ b/libraries/management/commands/import_library_version_docs_urls.py @@ -10,19 +10,16 @@ from versions.models import Version @click.command() @click.option( "--version", - type=str, - is_flag=False, help="Boost version number (example: 1.81.0). If a partial version number is " "provided, the command process all versions that contain the partial version " "number (example: '--version='1.7' would process 1.7.0, 1.7.1, 1.7.2, etc.)", ) @click.option( "--min-version", - type=str, default=settings.MINIMUM_BOOST_VERSION, help="Minimum Boost version to process (default: 1.30.0)", ) -def command(version, min_version): +def command(version: str, min_version: str): """Cycles through all Versions in the database, and for each version gets the corresponding tag's .gitmodules. diff --git a/libraries/management/commands/import_library_versions.py b/libraries/management/commands/import_library_versions.py index a4cdcd48..4cfe750d 100644 --- a/libraries/management/commands/import_library_versions.py +++ b/libraries/management/commands/import_library_versions.py @@ -7,15 +7,14 @@ from versions.tasks import import_library_versions @click.command() -@click.option("--token", is_flag=False, help="Github API token") -@click.option("--release", is_flag=False, help="Boost version number (example: 1.81.0)") +@click.option("--token", help="Github API token") +@click.option("--release", help="Boost version number (example: 1.81.0)") @click.option( "--min-release", - type=str, default=settings.MINIMUM_BOOST_VERSION, help="Minimum Boost version to process (default: 1.31.0)", ) -def command(min_release, release, token): +def command(min_release: str, release: str, token: str): """Cycles through all Versions in the database, and for each version gets the corresponding tag's .gitmodules. diff --git a/libraries/models.py b/libraries/models.py index f3ebdecf..bf6d5164 100644 --- a/libraries/models.py +++ b/libraries/models.py @@ -11,7 +11,7 @@ from core.models import RenderedContent from core.tasks import adoc_to_html from .managers import CommitDataManager -from .utils import write_content_to_tempfile +from .utils import generate_random_string, write_content_to_tempfile class Category(models.Model): @@ -131,9 +131,27 @@ class Library(models.Model): return self.name def save(self, *args, **kwargs): - """Override the save method to confirm the slug is set (or set it)""" + """Override the save method to confirm the slug is set (or set it) + + We need the slug to be unique, but we want to intelligently make that happen, + because there are libraries (like Container Hash) that are more easily managed + as two records due to changes in the data between versions. + """ + # Generate slug based on name if not self.slug: - self.slug = slugify(self.name) + slug = slugify(self.name) + + # If there is a library with that slug, try a slug based on the key from the + # gitmodules file + if Library.objects.filter(slug=slug).exclude(pk=self.pk).exists(): + slug = slugify(self.key) + + # If that slug already exists, append a random string to the slug + if Library.objects.filter(slug=slug).exclude(pk=self.pk).exists(): + random_str = generate_random_string() + slug = f"{slug}-{random_str}" + + self.slug = slug return super().save(*args, **kwargs) def get_description(self, client, tag="develop"): diff --git a/libraries/tasks.py b/libraries/tasks.py index 158871a6..fa9c06b9 100644 --- a/libraries/tasks.py +++ b/libraries/tasks.py @@ -79,7 +79,12 @@ LIBRARY_DOCS_EXCEPTIONS = { } ], # FIXME: Load correct path for 1.60.0 and prior - "detail": [{"generator": generate_library_docs_url, "min_version": "boost_61_0"}], + "detail": [ + { + "generator": generate_library_docs_url, + "min_version": "boost_1_61_0", + }, + ], "dynamic-bitset": [ { "generator": generate_library_docs_url_double_nested_library_html, @@ -92,6 +97,20 @@ LIBRARY_DOCS_EXCEPTIONS = { "function-types": [ {"generator": generate_library_docs_url, "max_version": "boost_1_60_0"} ], + "functionalhash": [ + { + "generator": generate_library_docs_url_v4, + "max_version": "boost_1_66_0", + "alternate_slug": "hash", + } + ], + "functionaloverloaded-function": [ + { + "generator": generate_library_docs_url, + "max_version": "boost_1_60_0", + "alternate_slug": "functional/overloaded_function", + } + ], "graphparallel": [ { "generator": generate_library_docs_url, diff --git a/libraries/utils.py b/libraries/utils.py index 1c588587..84a0fdb9 100644 --- a/libraries/utils.py +++ b/libraries/utils.py @@ -1,5 +1,7 @@ from datetime import datetime from dateutil.relativedelta import relativedelta +import random +import string import structlog import tempfile @@ -163,6 +165,12 @@ def generate_library_docs_url_string_view(boost_url_slug, library_slug): return f"/doc/libs/{boost_url_slug}/libs/utility/doc/html/utility/utilities/{library_slug}.html" # noqa +def generate_random_string(length=4): + characters = string.ascii_letters + random_string = "".join(random.choice(characters) for _ in range(length)) + return random_string + + def version_within_range( version: str, min_version: str = None, max_version: str = None ): diff --git a/versions/tasks.py b/versions/tasks.py index c17dd4d5..be074f9e 100644 --- a/versions/tasks.py +++ b/versions/tasks.py @@ -194,6 +194,15 @@ LIBRARY_KEY_EXCEPTIONS = { } +@app.task +def import_all_library_versions(token=None, version_type="tag"): + """Run import_library_versions for all versions""" + for version in Version.objects.active(): + import_library_versions.delay( + version.name, token=token, version_type=version_type + ) + + @app.task def import_library_versions(version_name, token=None, version_type="tag"): """For a specific version, imports all LibraryVersions using GitHub data""" @@ -306,8 +315,9 @@ def import_library_versions(version_name, token=None, version_type="tag"): version=version, library=library, defaults={"data": lib_data} ) if not library.github_url: - pass - # # todo: handle this. Need a github_url for these. + github_data = client.get_repo(repo_slug=library_name) + library.github_url = github_data.get("html_url", "") + library.save() # Retrieve and store the docs url for each library-version in this release get_and_store_library_version_documentation_urls_for_version.delay(version.pk)