Import commits for minor versions (#1332)

- Fixes #1327
- Commit import will ignore patch and beta versions.

libraries/admin.py

@@ -9,6 +9,7 @@ from django.utils.safestring import mark_safe
 from django.shortcuts import redirect
 
 from libraries.forms import CreateReportForm, CreateReportFullForm
+from versions.models import Version
 from versions.tasks import import_all_library_versions
 from .models import (
     Category,
@@ -48,7 +49,7 @@ class CommitAdmin(admin.ModelAdmin):
         return my_urls + urls
 
     def update_commits(self, request):
-        update_commits.delay()
+        update_commits.delay(clean=True)
         self.message_user(
             request,
             """
@@ -237,7 +238,9 @@ class LibraryAdmin(admin.ModelAdmin):
 
     def get_commits_per_release(self, pk):
         return (
-            LibraryVersion.objects.filter(library_id=pk)
+            LibraryVersion.objects.filter(
+                library_id=pk, version__in=Version.objects.minor_versions()
+            )
             .annotate(count=Count("commit"), version_name=F("version__name"))
             .order_by("-version__name")
             .filter(count__gt=0)
@@ -273,7 +276,9 @@ class LibraryAdmin(admin.ModelAdmin):
 
     def get_new_contributor_counts(self, pk):
         return (
-            LibraryVersion.objects.filter(library_id=pk)
+            LibraryVersion.objects.filter(
+                library_id=pk, version__in=Version.objects.minor_versions()
+            )
             .annotate(
                 up_to_count=CommitAuthor.objects.filter(
                     commit__library_version__version__name__lte=OuterRef(

libraries/forms.py

@@ -155,9 +155,7 @@ class CreateReportForm(CreateReportFullForm):
     """Form for creating a report for a specific release."""
 
     version = ModelChoiceField(
-        queryset=Version.objects.active()
-        .exclude(name__in=["develop", "master", "head"])
-        .order_by("-name")
+        queryset=Version.objects.minor_versions().order_by("-version_array")
     )
 
     def __init__(self, *args, **kwargs):
@@ -205,11 +203,19 @@ class CreateReportForm(CreateReportFullForm):
 
     def _count_new_contributors(self, libraries, library_order):
         version = self.cleaned_data["version"]
+        version_lt = list(
+            Version.objects.minor_versions()
+            .filter(version_array__lt=version.cleaned_version_parts_int)
+            .values_list("id", flat=True)
+        )
+        version_lte = version_lt + [version.id]
         lt_subquery = LibraryVersion.objects.filter(
-            version__name__lt=version.name, library=OuterRef("id")
+            version__in=version_lt,
+            library=OuterRef("id"),
         ).values("id")
         lte_subquery = LibraryVersion.objects.filter(
-            version__name__lte=version.name, library=OuterRef("id")
+            version__in=version_lte,
+            library=OuterRef("id"),
         ).values("id")
         return sorted(
             list(

libraries/github.py

@@ -57,6 +57,12 @@ class ParsedCommit:
 
 
 def get_commit_data_for_repo_versions(key):
+    """Fetch commit data between minor versions (ignore patches).
+
+    Get commits from one x.x.0 release to the next x.x.0 release. Commits
+    to and from patches or beta versions are ignored.
+
+    """
     library = Library.objects.get(key=key)
     parser = re.compile(
         r"^commit (?P<sha>\w+)(?:\n(?P<merge>Merge).*)?\nAuthor: (?P<name>[^\<]+)"
@@ -84,8 +90,9 @@ def get_commit_data_for_repo_versions(key):
             else:
                 break
         versions = [""] + list(
-            Version.objects.filter(library_version__library__key=library.key)
-            .order_by("name")
+            Version.objects.minor_versions()
+            .filter(library_version__library__key=library.key)
+            .order_by("version_array")
             .values_list("name", flat=True)
         )
         for a, b in zip(versions, versions[1:]):
@@ -428,7 +435,7 @@ class LibraryUpdater:
         }
         with transaction.atomic():
             if clean:
-                Commit.objects.filter(library_versions__library=obj).delete()
+                Commit.objects.filter(library_version__library=obj).delete()
             for commit in get_commit_data_for_repo_versions(obj.key):
                 author = authors.get(commit.email, None)
                 if not author:

libraries/tasks.py

@@ -184,10 +184,10 @@ def update_libraries():
 
 
 @app.task
-def update_commits(token=None):
+def update_commits(token=None, clean=False):
     updater = LibraryUpdater(token=token)
     for library in Library.objects.all():
-        updater.update_commits(obj=library)
+        updater.update_commits(obj=library, clean=clean)
     logger.info("update_commits finished.")
 
 

libraries/views.py

@@ -298,7 +298,10 @@ class LibraryDetail(FormMixin, DetailView):
 
     def get_commit_data_by_release(self):
         qs = (
-            LibraryVersion.objects.filter(library=self.object)
+            LibraryVersion.objects.filter(
+                library=self.object,
+                version__in=Version.objects.minor_versions(),
+            )
             .annotate(count=Count("commit"), version_name=F("version__name"))
             .order_by("-version__name")
         )[:20]
@@ -427,6 +430,9 @@ class LibraryDetail(FormMixin, DetailView):
             library_version = LibraryVersion.objects.get(
                 library=self.object, version=version
             )
+            prev_versions = Version.objects.minor_versions().filter(
+                version_array__lt=version.cleaned_version_parts_int
+            )
             qs = CommitAuthor.objects.filter(
                 commit__library_version=library_version
             ).annotate(
@@ -434,7 +440,7 @@ class LibraryDetail(FormMixin, DetailView):
                     Commit.objects.filter(
                         author_id=OuterRef("id"),
                         library_version__in=LibraryVersion.objects.filter(
-                            version__name__lt=version.name, library=self.object
+                            version__in=prev_versions, library=self.object
                         ),
                     )
                 )
@@ -450,7 +456,10 @@ class LibraryDetail(FormMixin, DetailView):
 
     def get_previous_contributors(self, version, exclude=None):
         library_versions = LibraryVersion.objects.filter(
-            library=self.object, version__name__lt=version.name
+            library=self.object,
+            version__in=Version.objects.minor_versions().filter(
+                version_array__lt=version.cleaned_version_parts_int
+            ),
         )
         qs = (
             CommitAuthor.objects.filter(commit__library_version__in=library_versions)

templates/libraries/detail.html

@@ -202,6 +202,10 @@
       xaxis: {
         categories: [{% for commit_data in commit_data_by_release %}"{{ commit_data.release }}",{% endfor %}],
         position: 'top',
+        labels: {
+          rotate: -45,
+          rotateAlways: true,
+        },
         axisBorder: {
           show: false
         },

versions/managers.py

@@ -1,4 +1,7 @@
 from django.db import models
+from django.db.models import Func, Value
+from django.db.models.functions import Replace
+from django.contrib.postgres.fields import ArrayField
 
 
 class VersionQuerySet(models.QuerySet):
@@ -22,6 +25,34 @@ class VersionQuerySet(models.QuerySet):
         old ones are generally deleted. But just in case."""
         return self.active().filter(beta=True).order_by("-name").first()
 
+    def with_version_split(self):
+        """Separates name into an array of [major, minor, patch].
+
+        Anything not matching the regex is removed from the queryset.
+
+        Example:
+            name = boost-1.85.0
+            version_array -> [1, 85, 0]
+            major -> 1
+            minor -> 85
+            patch -> 0
+
+        """
+        return self.filter(name__regex=r"^(boost-)?\d+\.\d+\.\d+$").annotate(
+            simple_version=Replace("name", Value("boost-"), Value("")),
+            version_array=Func(
+                "simple_version",
+                Value(r"\."),
+                function="regexp_split_to_array",
+                template="(%(function)s(%(expressions)s)::int[])",
+                arity=2,
+                output_field=ArrayField(models.IntegerField()),
+            ),
+            major=models.F("version_array__0"),
+            minor=models.F("version_array__1"),
+            patch=models.F("version_array__2"),
+        )
+
 
 class VersionManager(models.Manager):
     def get_queryset(self):
@@ -39,6 +70,14 @@ class VersionManager(models.Manager):
         """Return most recent active beta version"""
         return self.get_queryset().most_recent_beta()
 
+    def minor_versions(self):
+        """Filters for versions with patch = 0
+
+        Beta versions are removed.
+
+        """
+        return self.get_queryset().with_version_split().filter(patch=0)
+
     def version_dropdown(self):
         """Return the versions that should show in the version drop-down"""
         all_versions = self.active().filter(beta=False)

versions/models.py

@@ -107,6 +107,13 @@ class Version(models.Model):
         cleaned = re.sub(r"^[^0-9]*", "", self.name).split("beta")[0]
         return [part for part in cleaned.split(".") if part]
 
+    @cached_property
+    def cleaned_version_parts_int(self):
+        """Returns the release data as a list of integers."""
+        if not self.cleaned_version_parts:
+            return []
+        return [int(x) if x.isdigit() else 0 for x in self.cleaned_version_parts]
+
     @cached_property
     def release_notes_cache_key(self):
         """Returns the cahe key used to access the release notes in the