Correct release report added counts (#1865)

libraries/forms.py

@@ -751,6 +751,10 @@ class CreateReportForm(CreateReportFullForm):
         new_libraries = libraries.exclude(
             library_version__version__release_date__lte=prior_version.release_date
         ).prefetch_related("authors")
+        # TODO: we may in future need to find a way to show the removed libraries, for
+        #  now it's not needed. In that case the distinction between running this on a
+        #  ReportConfiguration with a real 'version' entry vs one that instead uses 'master'
+        #  will need to be considered
         top_contributors = self._get_top_contributors_for_version(version)
         # total messages sent during this release (version)
         total_mailinglist_count = EmailData.objects.filter(version=version).aggregate(
@@ -769,24 +773,16 @@ class CreateReportForm(CreateReportFullForm):
             commit_contributors_release_count,
             commit_contributors_new_count,
         ) = self._count_commit_contributors_totals(version, prior_version)
-        library_count = LibraryVersion.objects.filter(
-            version=version,
-            library__in=self.library_queryset,
-        ).count()
-        if prior_version:
-            library_count_prior = LibraryVersion.objects.filter(
-                version=prior_version,
-                library__in=self.library_queryset,
-            ).count()
-        else:
-            library_count_prior = 0
 
-        added_library_count = max(0, library_count - library_count_prior)
-        removed_library_count = max(0, library_count_prior - library_count)
+        added_library_count = new_libraries.count()
+        # TODO: connected to above todo, add removed_libraries.count()
+        removed_library_count = 0
+
         lines_added = LibraryVersion.objects.filter(
             version=version,
             library__in=self.library_queryset,
         ).aggregate(lines=Sum("insertions"))["lines"]
+
         lines_removed = LibraryVersion.objects.filter(
             version=version,
             library__in=self.library_queryset,
@@ -856,8 +852,7 @@ class CreateReportForm(CreateReportFullForm):
             "new_libraries": new_libraries,
             "batched_library_data": batched_library_data,
             "top_libraries_for_version": top_libraries_for_version,
-            "library_count": library_count,
-            "library_count_prior": library_count_prior,
+            "library_count": libraries.count(),
             "library_index_libraries": library_index_library_data,
             "added_library_count": added_library_count,
             "removed_library_count": removed_library_count,

libraries/github.py

@@ -204,7 +204,7 @@ class LibraryUpdater:
         # of other modules. Identified by the key used in the libraries.json file.
         self.skip_libraries = ["chrono/stopwatch"]
 
-    def get_library_list(self, gitmodules=None):
+    def get_library_list(self, gitmodules):
         """
         Retrieve the full list of library data for Boost libraries from their Github
         repos.

libraries/migrations/0031_remove_library_active_development.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.16 on 2025-08-08 21:56
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("libraries", "0030_commitauthor_user"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="library",
+            name="active_development",
+        ),
+    ]

libraries/models.py

@@ -192,7 +192,6 @@ class Library(models.Model):
     cpp_standard_minimum = models.CharField(
         max_length=50, blank=True, null=True
     )  # deprecated for LibraryVersion.cpp_standard_minimum
-    active_development = models.BooleanField(default=True, db_index=True)
     categories = models.ManyToManyField(Category, related_name="libraries")
 
     authors = models.ManyToManyField("users.User", related_name="authors")

versions/tasks.py

@@ -244,6 +244,19 @@ def skip_library_version(library_slug, version_slug):
     return False
 
 
+@app.task
+def gc_removed_submodules(library_keys: list[str], branch: str) -> None:
+    """Remove libraries that are not in the library_keys from the
+    library_versions list for the current version."""
+    library_version_keys = LibraryVersion.objects.filter(
+        version__name=branch
+    ).values_list("library__key", flat=True)
+    for k in library_version_keys:
+        if k not in library_keys:
+            LibraryVersion.objects.filter(version__name=branch, library__key=k).delete()
+            logger.info(f"{k=} library_version link to {branch=} garbage collected.")
+
+
 @app.task
 def import_library_versions(version_name, token=None, version_type="tag"):
     """For a specific version, imports all LibraryVersions using GitHub data"""
@@ -280,6 +293,7 @@ def import_library_versions(version_name, token=None, version_type="tag"):
 
     # For each gitmodule, gets its libraries.json file and save the libraries
     # to the version
+    library_keys = []
     for gitmodule in gitmodules:
         library_name = gitmodule["module"]
         if library_name in updater.skip_modules:
@@ -336,6 +350,9 @@ def import_library_versions(version_name, token=None, version_type="tag"):
         for lib_data in parsed_libraries:
             if lib_data["key"] in updater.skip_libraries:
                 continue
+            # tracking this 'key' because the gitmodule name doesn't directly match,
+            # e.g. interval in gitmodule, numericinterval in db/here
+            library_keys.append(lib_data["key"])
 
             # Handle exceptions based on version and library key
             exceptions = LIBRARY_KEY_EXCEPTIONS.get(lib_data["key"], [])
@@ -371,6 +388,12 @@ def import_library_versions(version_name, token=None, version_type="tag"):
                 library.github_url = github_data.get("html_url", "")
                 library.save()
 
+    # For any libraries no longer in gitmodules we want to remove master and develop
+    #  references from the library_versions list.
+    if version_name in ["master", "develop"]:
+        logger.info("Triggering  removed submodules garbage collection")
+        gc_removed_submodules.delay(library_keys, version_name)
+
     # Retrieve and store the docs url for each library-version in this release
     get_and_store_library_version_documentation_urls_for_version.delay(version.pk)