diff --git a/libraries/forms.py b/libraries/forms.py index 2f6d2dfc..c4681d6c 100644 --- a/libraries/forms.py +++ b/libraries/forms.py @@ -751,6 +751,10 @@ class CreateReportForm(CreateReportFullForm): new_libraries = libraries.exclude( library_version__version__release_date__lte=prior_version.release_date ).prefetch_related("authors") + # TODO: we may in future need to find a way to show the removed libraries, for + # now it's not needed. In that case the distinction between running this on a + # ReportConfiguration with a real 'version' entry vs one that instead uses 'master' + # will need to be considered top_contributors = self._get_top_contributors_for_version(version) # total messages sent during this release (version) total_mailinglist_count = EmailData.objects.filter(version=version).aggregate( @@ -769,24 +773,16 @@ class CreateReportForm(CreateReportFullForm): commit_contributors_release_count, commit_contributors_new_count, ) = self._count_commit_contributors_totals(version, prior_version) - library_count = LibraryVersion.objects.filter( - version=version, - library__in=self.library_queryset, - ).count() - if prior_version: - library_count_prior = LibraryVersion.objects.filter( - version=prior_version, - library__in=self.library_queryset, - ).count() - else: - library_count_prior = 0 - added_library_count = max(0, library_count - library_count_prior) - removed_library_count = max(0, library_count_prior - library_count) + added_library_count = new_libraries.count() + # TODO: connected to above todo, add removed_libraries.count() + removed_library_count = 0 + lines_added = LibraryVersion.objects.filter( version=version, library__in=self.library_queryset, ).aggregate(lines=Sum("insertions"))["lines"] + lines_removed = LibraryVersion.objects.filter( version=version, library__in=self.library_queryset, @@ -856,8 +852,7 @@ class CreateReportForm(CreateReportFullForm): "new_libraries": new_libraries, "batched_library_data": batched_library_data, "top_libraries_for_version": top_libraries_for_version, - "library_count": library_count, - "library_count_prior": library_count_prior, + "library_count": libraries.count(), "library_index_libraries": library_index_library_data, "added_library_count": added_library_count, "removed_library_count": removed_library_count, diff --git a/libraries/github.py b/libraries/github.py index 102290de..08954a4e 100644 --- a/libraries/github.py +++ b/libraries/github.py @@ -204,7 +204,7 @@ class LibraryUpdater: # of other modules. Identified by the key used in the libraries.json file. self.skip_libraries = ["chrono/stopwatch"] - def get_library_list(self, gitmodules=None): + def get_library_list(self, gitmodules): """ Retrieve the full list of library data for Boost libraries from their Github repos. diff --git a/libraries/migrations/0031_remove_library_active_development.py b/libraries/migrations/0031_remove_library_active_development.py new file mode 100644 index 00000000..0bf07a86 --- /dev/null +++ b/libraries/migrations/0031_remove_library_active_development.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.16 on 2025-08-08 21:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("libraries", "0030_commitauthor_user"), + ] + + operations = [ + migrations.RemoveField( + model_name="library", + name="active_development", + ), + ] diff --git a/libraries/models.py b/libraries/models.py index 0d3173bd..d5fa43fc 100644 --- a/libraries/models.py +++ b/libraries/models.py @@ -192,7 +192,6 @@ class Library(models.Model): cpp_standard_minimum = models.CharField( max_length=50, blank=True, null=True ) # deprecated for LibraryVersion.cpp_standard_minimum - active_development = models.BooleanField(default=True, db_index=True) categories = models.ManyToManyField(Category, related_name="libraries") authors = models.ManyToManyField("users.User", related_name="authors") diff --git a/versions/tasks.py b/versions/tasks.py index 95c57c56..4ed2debd 100644 --- a/versions/tasks.py +++ b/versions/tasks.py @@ -244,6 +244,19 @@ def skip_library_version(library_slug, version_slug): return False +@app.task +def gc_removed_submodules(library_keys: list[str], branch: str) -> None: + """Remove libraries that are not in the library_keys from the + library_versions list for the current version.""" + library_version_keys = LibraryVersion.objects.filter( + version__name=branch + ).values_list("library__key", flat=True) + for k in library_version_keys: + if k not in library_keys: + LibraryVersion.objects.filter(version__name=branch, library__key=k).delete() + logger.info(f"{k=} library_version link to {branch=} garbage collected.") + + @app.task def import_library_versions(version_name, token=None, version_type="tag"): """For a specific version, imports all LibraryVersions using GitHub data""" @@ -280,6 +293,7 @@ def import_library_versions(version_name, token=None, version_type="tag"): # For each gitmodule, gets its libraries.json file and save the libraries # to the version + library_keys = [] for gitmodule in gitmodules: library_name = gitmodule["module"] if library_name in updater.skip_modules: @@ -336,6 +350,9 @@ def import_library_versions(version_name, token=None, version_type="tag"): for lib_data in parsed_libraries: if lib_data["key"] in updater.skip_libraries: continue + # tracking this 'key' because the gitmodule name doesn't directly match, + # e.g. interval in gitmodule, numericinterval in db/here + library_keys.append(lib_data["key"]) # Handle exceptions based on version and library key exceptions = LIBRARY_KEY_EXCEPTIONS.get(lib_data["key"], []) @@ -371,6 +388,12 @@ def import_library_versions(version_name, token=None, version_type="tag"): library.github_url = github_data.get("html_url", "") library.save() + # For any libraries no longer in gitmodules we want to remove master and develop + # references from the library_versions list. + if version_name in ["master", "develop"]: + logger.info("Triggering removed submodules garbage collection") + gc_removed_submodules.delay(library_keys, version_name) + # Retrieve and store the docs url for each library-version in this release get_and_store_library_version_documentation_urls_for_version.delay(version.pk)