boost/website-v2
2
0
Fork 0
mirror of https://github.com/boostorg/website-v2.git synced 2026-01-19 04:42:17 +00:00
Code Issues Projects Releases Wiki Activity

Import commits per release and create release reports (#1263)

Browse Source 
View stats per release, we do this by
doing log diffs between release tags. Ex: `git log
boost-1.78.0..boost-1.79.0`. The output is parsed and the commits
are saved with a foreign key to the `LibraryVersion` it relates to.

- commits are imported by doing "bare" clones (no project files, only
git data) of repos into temporary directories, as created by python's
bulitin `tempfile.TemporaryDirectory`
- Added Commit model
- Added CommitAuthor model
- Added CommitAuthorEmail model
  - One CommitAuthor can have many emails.
- Added task for importing commits. (and admin link to trigger it)
- Added task for importing CommitAuthor github data (avatar and profile
url, with admin link to trigger it)
- Added a basic Library stat page which can be viewed by going to the
admin -> library -> view stats.
- Added a `Get Release Report` button in the `LibraryAdmin` which allows
a staff member to select a boost version and up to 8 libraries to
generate a report for. The report is just a webpage which attempts to
convert cleanly to a pdf using the browser's print to pdf functionality.
- Updated the Library Detail page to show commits per release instead of
per month.
- Updated the Library Detail page to show `Maintainers & Contributors`
sorted by maintainers, then the top contributors for the selected
release, then the top contributors overall by commits descending.
- Removed CommitData, which was tracking monthly commit stats
This commit is contained in:
Brian Perrett
2024-09-25 15:09:07 -07:00
committed by GitHub
parent 6d3e82f8e4
commit 48c09d3d5e
28 changed files with 1435 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
core/githubhelper.py
View File

@@ -3,6 +3,9 @@ import os
import re
from collections import defaultdict
from datetime import datetime
from socket import gaierror
import time
from urllib.error import URLError
import requests
import structlog
@@ -67,6 +70,26 @@ class GithubAPIClient:
token = os.environ.get("GITHUB_TOKEN", None)
return GhApi(token=token)
def with_retry(self, fn, retry_count=5):
count = 0
while count < 5:
count += 1
try:
output = fn()
except URLError as e:
if getattr(e, "args", None) and isinstance(
e.args[0], gaierror
): # connection error
if count == retry_count:
raise e
self.logger.warning(f"URLError: {e}")
self.logger.info(f"Retry backoff {2**count} seconds.")
time.sleep(2**count)
else:
raise e
else:
return output
def get_blob(self, repo_slug: str = None, file_sha: str = None) -> dict:
"""
Get the blob from the GitHub API.
@@ -89,6 +112,14 @@ class GithubAPIClient:
owner=self.owner, repo=repo_slug, commit_sha=commit_sha
)
def get_repo_ref(self, repo_slug: str = None, ref: str = None) -> dict:
"""Get a repo commit by ref."""
if not repo_slug:
repo_slug = self.repo_slug
return self.with_retry(
lambda: self.api.repos.get_commit(owner=self.owner, repo=repo_slug, ref=ref)
)
def get_commits(
self,
repo_slug: str = None,
@@ -132,6 +163,58 @@ class GithubAPIClient:
return all_commits
def compare(
self,
repo_slug: str = None,
ref_from: str = None,
ref_to: str = None,
):
"""Compare and get commits between 2 refs.
:param repo_slug: str, the repository slug. If not provided, the class
instance's repo_slug will be used.
:param ref_from: str, the ref to start from.
:param ref_to: str, the ref to end with.
:return: List[ComparePage], list of the pages returned by compare_commits
"""
repo_slug = repo_slug or self.repo_slug
# Get the commits
all_pages = []
basehead = f"{ref_from}...{ref_to}"
try:
page = 1
per_page = 100
while True:
output = self.with_retry(
lambda: self.api.repos.compare_commits(
owner=self.owner,
repo=repo_slug,
basehead=basehead,
per_page=per_page,
page=page,
)
)
if not output:
break
all_pages.append(output)
if len(output["commits"]) < per_page:
break
page += 1
except Exception as e:
self.logger.exception(
"compare refs failed",
repo=repo_slug,
ref_from=ref_from,
ref_to=ref_to,
exc_msg=str(e),
)
return []
return all_pages
def get_first_tag(self, repo_slug: str = None):
"""
Retrieves the earliest tag in the repo.

4
core/management/commands/boost_setup.py
View File

@@ -33,10 +33,6 @@ def command(token):
call_command("update_authors")
click.secho("Finished adding library authors.", fg="green")
click.secho("Importing library commit history...", fg="green")
call_command("import_commit_counts", "--token", token)
click.secho("Finished importing library commit history.", fg="green")
click.secho("Importing most recent beta version...", fg="green")
call_command("import_beta_release", "--token", token, "--delete-versions")
click.secho("Finished importing most recent beta version.", fg="green")

14
docs/commands.md
View File

@@ -10,7 +10,7 @@
- [`import_library_version_docs_urls`](#import_library_version_docs_urls)
- [`update_maintainers`](#update_maintainers)
- [`update_authors`](#update_authors)
- [`import_commit_counts`](#import_commit_counts)
- [`import_commits`](#import_commits)
- [`import_beta_release`](#import_beta_release)
## `boost_setup`
@@ -209,24 +209,22 @@ If both the `--release` and the `--library-name` are passed, the command will lo
| `--library-name` | string | Name of the library. If passed, the command will load maintainers for only this library. |
## `import_commits`
## `import_commit_counts`
**Purpose**: Saves `CommitData` objects. Each object contains the data for the number of commits made to the `master` branch of a given `Library` with in a given month.
**Purpose**: Cycles through all libraries and their library versions to import `Commit`, `CommitAuthor`, and `CommitAuthorEmail` models. Updates `CommitAuthor` github profile URLs and avatar URLs.
**Example**
```bash
./manage.py import_commit_counts
./manage.py import_commits
```
**Options**
| Options | Format | Description |
|----------------------|--------|--------------------------------------------------------------|
| `--branch` | string | Specify the branch you want to count commits for. Defaults to `master`. |
| `--token` | string | Pass a GitHub API token. If not passed, will use the value in `settings.GITHUB_TOKEN`. |
| `--key` | string | Key of the library. If passed, the command will import commits for only this library. |
| `--clean` | boolean | If passed, will delete all existing commits before importing new ones. |
## `import_beta_release`

4
docs/first_time_data_import.md
View File

@@ -34,7 +34,7 @@ The `boost_setup` command will run all of the processes listed here:
# Save other data we need for Libraries and LibraryVersions
./manage.py update_maintainers
./manage.py update_authors
./manage.py import_commit_counts
./manage.py import_commits
# Get the most recent beta release, and delete old beta releases
./manage.py import_beta_release --delete-versions
@@ -49,7 +49,7 @@ Collectively, this is what these management commands accomplish:
3. `import_library_versions`: Establishes which Boost libraries are included in which Boost versions. That information is stored in `LibraryVersion` objects. This process also stores the link to the version-specific Boost documentation for this library.
4. `update_maintainers`: For each `LibraryVersion`, saves the maintainers as `User` objects and makes sure they are associated with the `LibraryVersion`.
5. `update_authors`: For each `Library`, saves the authors as `User` objects and makes sure they are associated with the `Library`.
6. `import_commit_counts`: For each `Library`, uses information in the GitHub API to save the last 12 months of commit history. One `CommitData` object per library, per month is created to store the number of commits to the `master` branch of that library for that month.
6. `import_commits`: For each `Library`, iterate through the `LibraryVersion`s and create `Commit`, `CommitAuthor`, and `CommitAuthorEmail` objects. Also attempts to update `CommitAuthor`s with their github profile URL and Avatar URL.
7. `import_beta_release`: Retrieves the most recent beta release from GitHub and imports it. If `--delete-versions` is passed, will delete the existing beta releases in the database.
## Further Reading

283
libraries/admin.py
View File

@@ -1,17 +1,111 @@
from django.contrib import admin
from django.db import transaction
from django.db.models import F, Count, OuterRef, Window
from django.db.models.functions import RowNumber
from django.http import HttpResponseRedirect
from django.template.response import TemplateResponse
from django.urls import path, reverse
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.shortcuts import redirect
from libraries.forms import CreateReportForm
from versions.tasks import import_all_library_versions
from .models import Category, CommitData, Issue, Library, LibraryVersion, PullRequest
from .models import (
Category,
Commit,
CommitAuthor,
CommitAuthorEmail,
Issue,
Library,
LibraryVersion,
PullRequest,
)
from .tasks import (
update_commit_counts,
update_commit_author_github_data,
update_commits,
update_libraries,
update_library_version_documentation_urls_all_versions,
)
@admin.register(Commit)
class CommitAdmin(admin.ModelAdmin):
list_display = ["library_version", "sha", "author"]
autocomplete_fields = ["author", "library_version"]
list_filter = ["library_version__library"]
search_fields = ["sha", "author__name"]
change_list_template = "admin/commit_change_list.html"
def get_urls(self):
urls = super().get_urls()
my_urls = [
path(
"update_commits/",
self.admin_site.admin_view(self.update_commits),
name="update_commits",
),
]
return my_urls + urls
def update_commits(self, request):
update_commits.delay()
self.message_user(
request,
"""
Commits for all libraries are being imported.
""",
)
return HttpResponseRedirect("../")
class CommitAuthorEmailInline(admin.TabularInline):
model = CommitAuthorEmail
extra = 0
@admin.register(CommitAuthor)
class CommitAuthorAdmin(admin.ModelAdmin):
search_fields = ["name"]
actions = ["merge_authors"]
inlines = [CommitAuthorEmailInline]
change_list_template = "admin/commit_author_change_list.html"
def get_urls(self):
urls = super().get_urls()
my_urls = [
path(
"update_github_data/",
self.admin_site.admin_view(self.update_github_data),
name="commit_author_update_github_data",
),
]
return my_urls + urls
def update_github_data(self, request):
update_commit_author_github_data.delay(clean=True)
self.message_user(
request,
"""
Updating CommitAuthor Github data.
""",
)
return HttpResponseRedirect("../")
@admin.action(
description="Combine 2 or more authors into one. References will be updated."
)
def merge_authors(self, request, queryset):
objects = list(queryset)
if len(objects) < 2:
return
author = objects[0]
with transaction.atomic():
for other in objects[1:]:
author.merge_author(other)
message = "Merged authors -- " + ", ".join([x.name for x in objects])
self.message_user(request, message)
@admin.register(Category)
class CategoryAdmin(admin.ModelAdmin):
list_display = ["name"]
@@ -19,71 +113,6 @@ class CategoryAdmin(admin.ModelAdmin):
search_fields = ["name"]
@admin.register(CommitData)
class CommitDataAdmin(admin.ModelAdmin):
list_display = (
"library",
"commit_count_formatted",
"month_year_formatted",
"branch",
"library_link",
)
list_filter = ("library__name", "branch", "month_year")
search_fields = ("library__name", "branch")
date_hierarchy = "month_year"
ordering = ("library__name", "-month_year")
autocomplete_fields = ["library"]
change_list_template = "admin/commit_data_change_list.html"
def commit_count_formatted(self, obj):
return f"{obj.commit_count:,}"
commit_count_formatted.admin_order_field = "commit_count"
commit_count_formatted.short_description = "Commit Count"
def month_year_formatted(self, obj):
return obj.month_year.strftime("%B %Y")
month_year_formatted.admin_order_field = "month_year"
month_year_formatted.short_description = "Month/Year"
def library_link(self, obj):
return format_html(
'<a href="{}">{}</a>',
reverse("admin:libraries_library_change", args=(obj.library.pk,)),
obj.library.name,
)
library_link.short_description = "Library Details"
def formfield_for_foreignkey(self, db_field, request, **kwargs):
if db_field.name == "library":
kwargs["queryset"] = Library.objects.order_by("name")
return super().formfield_for_foreignkey(db_field, request, **kwargs)
def get_urls(self):
urls = super().get_urls()
my_urls = [
path(
"update_commit_data/",
self.update_commit_data,
name="update_commit_data",
),
]
return my_urls + urls
def update_commit_data(self, request):
"""Run the task to refresh the library data from GitHub"""
update_commit_counts.delay()
self.message_user(
request,
"""
Commit data is being refreshed.
""",
)
return HttpResponseRedirect("../")
class LibraryVersionInline(admin.TabularInline):
model = LibraryVersion
extra = 0
@@ -93,7 +122,7 @@ class LibraryVersionInline(admin.TabularInline):
@admin.register(Library)
class LibraryAdmin(admin.ModelAdmin):
list_display = ["name", "key", "github_url"]
list_display = ["name", "key", "github_url", "view_stats"]
search_fields = ["name", "description"]
list_filter = ["categories"]
ordering = ["name"]
@@ -104,9 +133,51 @@ class LibraryAdmin(admin.ModelAdmin):
urls = super().get_urls()
my_urls = [
path("update_libraries/", self.update_libraries, name="update_libraries"),
path(
"<int:pk>/stats/",
self.admin_site.admin_view(self.library_stat_detail),
name="library_stat_detail",
),
path(
"report-form/",
self.admin_site.admin_view(self.report_form_view),
name="library_report_form",
),
path(
"report/",
self.admin_site.admin_view(self.report_view),
name="library_report",
),
]
return my_urls + urls
def report_form_view(self, request):
form = CreateReportForm()
context = {}
if request.GET.get("version", None):
form = CreateReportForm(request.GET)
if form.is_valid():
context.update(form.get_stats())
return redirect(
reverse("admin:library_report") + f"?{request.GET.urlencode()}"
)
if not context:
context["form"] = form
return TemplateResponse(request, "admin/library_report_form.html", context)
def report_view(self, request):
form = CreateReportForm(request.GET)
context = {"form": form}
if form.is_valid():
context.update(form.get_stats())
else:
return redirect("admin:library_report_form")
return TemplateResponse(request, "admin/library_report_detail.html", context)
def view_stats(self, instance):
url = reverse("admin:library_stat_detail", kwargs={"pk": instance.pk})
return mark_safe(f"<a href='{url}'>View Stats</a>")
def update_libraries(self, request):
"""Run the task to refresh the library data from GitHub"""
update_libraries.delay()
@@ -119,6 +190,80 @@ class LibraryAdmin(admin.ModelAdmin):
)
return HttpResponseRedirect("../")
def library_stat_detail(self, request, pk):
context = {
"object": self.get_object(request, pk),
"commits_per_release": self.get_commits_per_release(pk),
"commits_per_author": self.get_commits_per_author(pk),
"commits_per_author_release": self.get_commits_per_author_release(pk),
"new_contributor_counts": self.get_new_contributor_counts(pk),
}
return TemplateResponse(request, "admin/library_stat_detail.html", context)
def get_commits_per_release(self, pk):
return (
LibraryVersion.objects.filter(library_id=pk)
.annotate(count=Count("commit"), version_name=F("version__name"))
.order_by("-version__name")
.filter(count__gt=0)
)[:10]
def get_commits_per_author(self, pk):
return (
CommitAuthor.objects.filter(commit__library_version__library_id=pk)
.annotate(count=Count("commit"))
.order_by("-count")[:20]
)
def get_commits_per_author_release(self, pk):
return (
LibraryVersion.objects.filter(library_id=pk)
.filter(commit__author__isnull=False)
.annotate(
count=Count("commit"),
row_number=Window(
expression=RowNumber(), partition_by=["id"], order_by=["-count"]
),
)
.values(
"count",
"commit__author",
"commit__author__name",
"version__name",
"commit__author__avatar_url",
)
.order_by("-version__name", "-count")
.filter(row_number__lte=3)
)
def get_new_contributor_counts(self, pk):
return (
LibraryVersion.objects.filter(library_id=pk)
.annotate(
up_to_count=CommitAuthor.objects.filter(
commit__library_version__version__name__lte=OuterRef(
"version__name"
),
commit__library_version__library_id=pk,
)
.values("commit__library_version__library")
.annotate(count=Count("id", distinct=True))
.values("count")[:1],
before_count=CommitAuthor.objects.filter(
commit__library_version__version__name__lt=OuterRef(
"version__name"
),
commit__library_version__library_id=pk,
)
.values("commit__library_version__library")
.annotate(count=Count("id", distinct=True))
.values("count")[:1],
count=F("up_to_count") - F("before_count"),
)
.order_by("-version__name")
.select_related("version")
)
@admin.register(LibraryVersion)
class LibraryVersionAdmin(admin.ModelAdmin):

202
libraries/forms.py
View File

@@ -1,7 +1,8 @@
from django.db.models import F, Q, Count, OuterRef
from django.forms import Form, ModelChoiceField, ModelForm
from versions.models import Version
from .models import Library
from .models import Commit, CommitAuthor, Library, LibraryVersion
class LibraryForm(ModelForm):
@@ -19,3 +20,202 @@ class VersionSelectionForm(Form):
label="Select a version",
empty_label="Choose a version...",
)
class CreateReportForm(Form):
library_queryset = Library.objects.all().order_by("name")
version = ModelChoiceField(
queryset=Version.objects.active()
.exclude(name__in=["develop", "master", "head"])
.order_by("-name")
)
library_1 = ModelChoiceField(
queryset=library_queryset,
required=False,
help_text=(
"If none are selected, the top 5 for this release will be auto-selected."
),
)
library_2 = ModelChoiceField(
queryset=library_queryset,
required=False,
)
library_3 = ModelChoiceField(
queryset=library_queryset,
required=False,
)
library_4 = ModelChoiceField(
queryset=library_queryset,
required=False,
)
library_5 = ModelChoiceField(
queryset=library_queryset,
required=False,
)
library_6 = ModelChoiceField(
queryset=library_queryset,
required=False,
)
library_7 = ModelChoiceField(
queryset=library_queryset,
required=False,
)
library_8 = ModelChoiceField(
queryset=library_queryset,
required=False,
)
def _get_top_contributors_for_version(self):
return (
CommitAuthor.objects.filter(
commit__library_version__version=self.cleaned_data["version"]
)
.annotate(commit_count=Count("commit"))
.values("name", "avatar_url", "commit_count")
.order_by("-commit_count")[:10]
)
def _get_top_libraries_for_version(self):
return (
Library.objects.filter(
library_version=LibraryVersion.objects.filter(
library=OuterRef("id"), version=self.cleaned_data["version"]
)[:1],
)
.annotate(commit_count=Count("library_version__commit"))
.order_by("-commit_count")[:5]
)
def _get_library_order(self, top_libraries_release):
library_order = [
x.id
for x in [
self.cleaned_data["library_1"],
self.cleaned_data["library_2"],
self.cleaned_data["library_3"],
self.cleaned_data["library_4"],
self.cleaned_data["library_5"],
self.cleaned_data["library_6"],
self.cleaned_data["library_7"],
self.cleaned_data["library_8"],
]
if x is not None
]
if not library_order:
library_order = [x.id for x in top_libraries_release]
return library_order
def _get_library_full_counts(self, libraries, library_order):
return sorted(
list(
libraries.annotate(
commit_count=Count("library_version__commit")
).values("commit_count", "id")
),
key=lambda x: library_order.index(x["id"]),
)
def _get_library_version_counts(self, libraries, library_order):
return sorted(
list(
libraries.filter(
library_version=LibraryVersion.objects.filter(
library=OuterRef("id"), version=self.cleaned_data["version"]
)[:1]
)
.annotate(commit_count=Count("library_version__commit"))
.values("commit_count", "id")
),
key=lambda x: library_order.index(x["id"]),
)
def _count_new_contributors(self, libraries, library_order):
version = self.cleaned_data["version"]
lt_subquery = LibraryVersion.objects.filter(
version__name__lt=version.name, library=OuterRef("id")
).values("id")
lte_subquery = LibraryVersion.objects.filter(
version__name__lte=version.name, library=OuterRef("id")
).values("id")
return sorted(
list(
libraries.annotate(
authors_before_release_count=Count(
"library_version__commit__author",
filter=Q(library_version__in=lt_subquery),
distinct=True,
),
authors_through_release_count=Count(
"library_version__commit__author",
filter=Q(library_version__in=lte_subquery),
distinct=True,
),
)
.annotate(
count=F("authors_through_release_count")
- F("authors_before_release_count")
)
.values("id", "count")
),
key=lambda x: library_order.index(x["id"]),
)
def _get_top_contributors_for_library_version(self, library_order):
top_contributors_release = []
for library_id in library_order:
top_contributors_release.append(
CommitAuthor.objects.filter(
commit__library_version=LibraryVersion.objects.get(
version=self.cleaned_data["version"], library_id=library_id
)
)
.annotate(commit_count=Count("commit"))
.values(
"name",
"avatar_url",
"commit_count",
"commit__library_version__library_id",
)
.order_by("-commit_count")[:10]
)
return top_contributors_release
def get_stats(self):
version = self.cleaned_data["version"]
commit_count = Commit.objects.filter(
library_version__version__name__lte=version.name
).count()
version_commit_count = Commit.objects.filter(
library_version__version=version
).count()
top_libraries_for_version = self._get_top_libraries_for_version()
library_order = self._get_library_order(top_libraries_for_version)
libraries = Library.objects.filter(id__in=library_order)
library_data = [
{
"library": a,
"full_count": b,
"version_count": c,
"top_contributors_release": d,
"new_contributors_count": e,
}
for a, b, c, d, e in zip(
sorted(list(libraries), key=lambda x: library_order.index(x.id)),
self._get_library_full_counts(libraries, library_order),
self._get_library_version_counts(libraries, library_order),
self._get_top_contributors_for_library_version(library_order),
self._count_new_contributors(libraries, library_order),
)
]
top_contributors = self._get_top_contributors_for_version()
return {
"version": version,
"commit_count": commit_count,
"version_commit_count": version_commit_count,
"top_contributors_release_overall": top_contributors,
"library_data": library_data,
"top_libraries_for_version": top_libraries_for_version,
"library_count": Library.objects.all().count(),
}

252
libraries/github.py
View File

@@ -1,13 +1,33 @@
import structlog
import re
import time
import tempfile
from dataclasses import dataclass
from pathlib import Path
from dateutil.relativedelta import relativedelta
from django.contrib.auth import get_user_model
from django.utils import timezone
import structlog
from ghapi.core import HTTP404NotFoundError
from fastcore.xtras import obj2dict
from django.db.models import Exists, OuterRef
from django.utils.autoreload import subprocess
from django.contrib.auth import get_user_model
from django.db import transaction
from django.utils import dateparse, timezone
from versions.models import Version
from .models import (
Category,
Commit,
CommitAuthor,
CommitAuthorEmail,
Issue,
Library,
LibraryVersion,
PullRequest,
)
from core.githubhelper import GithubAPIClient, GithubDataParser
from .models import Category, Issue, Library, PullRequest
from .utils import generate_fake_email, parse_date
logger = structlog.get_logger()
@@ -24,6 +44,79 @@ FIRST_OF_CURRENT_MONTH = timezone.make_aware(
) - relativedelta(days=1)
@dataclass
class ParsedCommit:
email: str
name: str
message: str
sha: str
version: str
is_merge: bool
committed_at: timezone.datetime
avatar_url: str | None = None
def get_commit_data_for_repo_versions(key):
library = Library.objects.get(key=key)
parser = re.compile(
r"^commit (?P<sha>\w+)(?:\n(?P<merge>Merge).*)?\nAuthor: (?P<name>[^\<]+)"
r"\s+\<(?P<email>[^\>]+)\>\nDate:\s+(?P<date>.*)\n(?P<message>(.|\n)+?)"
r"(?=(commit|\Z))",
flags=re.MULTILINE,
)
retry_count = 0
with tempfile.TemporaryDirectory() as temp_dir:
git_dir = Path(temp_dir) / f"{library.key}.git"
while retry_count < 5:
retry_count += 1
completed = subprocess.run(
["git", "clone", f"{library.github_url}.git", "--bare", str(git_dir)],
capture_output=True,
)
error = completed.stderr.decode()
if "fatal: unable to access" in error:
logger.warning(
f"{completed.args} failed. Retrying git clone. Retry {retry_count}."
)
time.sleep(2**retry_count)
continue
else:
break
versions = [""] + list(
Version.objects.filter(library_version__library__key=library.key)
.order_by("name")
.values_list("name", flat=True)
)
for a, b in zip(versions, versions[1:]):
log_output = subprocess.run(
["git", "--git-dir", str(git_dir), "log", f"{a}..{b}", "--date", "iso"],
capture_output=True,
)
commits = log_output.stdout.decode()
for match in parser.finditer(commits):
groups = match.groupdict()
name = groups["name"].strip()
email = groups["email"].strip()
sha = groups["sha"].strip()
is_merge = bool(groups.get("merge", False))
message = groups["message"].strip("\n")
message = "\n".join(
[m[4:] if m.startswith(" ") else m for m in message.split("\n")]
)
committed_at = dateparse.parse_datetime(groups["date"])
assert committed_at # should always exist
yield ParsedCommit(
email=email,
name=name,
message=message,
sha=sha,
committed_at=committed_at,
is_merge=is_merge,
version=b,
)
class LibraryUpdater:
"""
This class is used to sync Libraries from the list of git submodules
@@ -219,54 +312,6 @@ class LibraryUpdater:
obj.maintainers.add(user)
self.logger.info(f"User {user.email} added as a maintainer of {obj}")
def update_monthly_commit_counts(
self,
branch: str = "master",
since=FIRST_OF_MONTH_ONE_YEAR_AGO,
until=FIRST_OF_CURRENT_MONTH,
):
"""Update the monthly commit data for all libraries
:param branch: Branch to update commit data for. Defaults to "master".
:param since: Year to update commit data for. Defaults to a year ago
:param until: Year to update commit data for. Defaults to present year
Note: Overrides CommitData objects for the library; does not increment
the count.
"""
self.logger.info("updating_monthly_commit_data")
for library in Library.objects.all():
self.update_monthly_commit_counts_for_library(
library, branch=branch, since=since, until=until
)
def update_monthly_commit_counts_for_library(
self,
obj,
branch: str = "master",
since=FIRST_OF_MONTH_ONE_YEAR_AGO,
until=FIRST_OF_CURRENT_MONTH,
):
"""Update the commit counts for a specific library."""
commits = self.client.get_commits(
repo_slug=obj.github_repo, branch=branch, since=since, until=until
)
commit_data = self.parser.get_commits_per_month(commits)
for month_year, commit_count in commit_data.items():
data_obj, created = obj.commit_data.update_or_create(
month_year=month_year,
branch=branch,
defaults={"commit_count": commit_count},
)
self.logger.info(
"commit_data_updated",
commit_data_pk=data_obj.pk,
obj_created=created,
library=obj.name,
branch=branch,
)
def update_issues(self, obj):
"""Import GitHub issues for the library and update the database"""
self.logger.info("updating_repo_issues")
@@ -370,3 +415,104 @@ class LibraryUpdater:
pr_github_id=pr_dict.get("id"),
exc_msg=str(e),
)
def update_commits(self, obj: Library, clean=False):
"""Import a record of all commits between LibraryVersions."""
authors = {}
commits = []
library_versions = {
x.version.name: x
for x in LibraryVersion.objects.filter(library=obj).select_related(
"version"
)
}
with transaction.atomic():
if clean:
Commit.objects.filter(library_versions__library=obj).delete()
for commit in get_commit_data_for_repo_versions(obj.key):
author = authors.get(commit.email, None)
if not author:
if (
commit_author_email := CommitAuthorEmail.objects.filter(
email=commit.email,
)
.select_related("author")
.first()
):
author = commit_author_email.author
else:
author = CommitAuthor.objects.create(
name=commit.name, avatar_url=commit.avatar_url
)
CommitAuthorEmail.objects.create(
email=commit.email, author=author
)
authors[commit.email] = author
commits.append(
Commit(
author=author,
library_version=library_versions[commit.version],
sha=commit.sha,
message=commit.message,
committed_at=commit.committed_at,
is_merge=commit.is_merge,
)
)
Commit.objects.bulk_create(
commits,
update_conflicts=True,
update_fields=["author", "message", "committed_at", "is_merge"],
unique_fields=["library_version", "sha"],
)
def update_commit_author_github_data(self, obj=None, email=None, overwrite=False):
"""Update CommitAuthor data by parsing data on their most recent commit."""
if email:
authors = CommitAuthor.objects.filter(
Exists(CommitAuthorEmail.objects.filter(id=OuterRef("pk"), email=email))
)
elif obj:
authors = CommitAuthor.objects.filter(
Exists(
Library.objects.filter(
library_version__commit__author=OuterRef("id"),
pk=obj.pk,
)
)
)
else:
authors = CommitAuthor.objects.all()
if not overwrite:
authors = authors.filter(avatar_url=None)
authors = authors.annotate(
most_recent_commit_sha=Commit.objects.filter(author=OuterRef("pk"))
.order_by("-committed_at")
.values("sha")[:1]
).annotate(
most_recent_library_key=Library.objects.filter(
library_version__commit__sha=OuterRef("most_recent_commit_sha")
).values("key")[:1]
)
libraries = Library.objects.filter(
key__in=[x.most_recent_library_key for x in authors]
)
repos = {x.key: x for x in libraries}
for author in authors:
try:
commit = self.client.get_repo_ref(
repo_slug=repos[author.most_recent_library_key].github_repo,
ref=author.most_recent_commit_sha,
)
except HTTP404NotFoundError:
self.logger.info(
f"Commit not found. Skipping avatar update for {author}."
)
continue
if gh_author := commit["author"]:
if gh_author["avatar_url"]:
author.avatar_url = gh_author["avatar_url"]
if gh_author["html_url"]:
author.github_profile_url = gh_author["html_url"]
author.save(update_fields=["avatar_url", "github_profile_url"])

16
libraries/management/commands/import_commit_counts.py
View File

@@ -1,16 +0,0 @@
import djclick as click
from libraries.tasks import update_commit_counts
@click.command()
@click.option("--token", is_flag=False, help="Github API token")
def command(token):
"""Imports commit counts for all libraries, broken down by month, and saves
them to the database. This is a one-time import.
:param token: Github API token
"""
click.secho("Importing library commit history...", fg="green")
update_commit_counts(token=token)
click.secho("Finished importing library commit history", fg="green")

21
libraries/management/commands/import_commits.py Normal file
View File

@@ -0,0 +1,21 @@
import djclick as click
from libraries.github import LibraryUpdater
from libraries.models import Library
@click.command()
@click.option("--key", is_flag=False, help="Library Key", default=None)
@click.option("--clean", is_flag=True, help="Library Key", default=False)
def command(key, clean):
updater = LibraryUpdater()
click.secho("Importing individual library commits...", fg="green")
if key is None:
for library in Library.objects.all():
click.secho(f"Importing commits for {library}")
updater.update_commits(library, clean=clean)
updater.update_commit_author_github_data()
else:
library = Library.objects.get(key=key)
updater.update_commits(library, clean=clean)
updater.update_commit_author_github_data(obj=library)
click.secho("Finished importing individual library commits.", fg="green")

21
libraries/management/commands/update_author_github_data.py Normal file
View File

@@ -0,0 +1,21 @@
import djclick as click
from libraries.github import LibraryUpdater
from libraries.models import Library
@click.command()
@click.option("--key", is_flag=False, help="Library Key", default=None)
@click.option("--clean", is_flag=True, help="Library Key", default=False)
def command(key, clean):
updater = LibraryUpdater()
click.secho(
"Updating author avatars from github. This may take a while, "
"depending on how many authors need to be updated...",
fg="green",
)
if key is None:
updater.update_commit_author_github_data(overwrite=clean)
else:
library = Library.objects.get(key=key)
updater.update_commit_author_github_data(obj=library, overwrite=clean)
click.secho("Finished updating author avatars from github...", fg="green")

35
libraries/managers.py
View File

@@ -1,35 +0,0 @@
import datetime
from dateutil.relativedelta import relativedelta
from django.db import models
from django.db.models import Sum
from django.db.models.functions import ExtractYear
class CommitDataManager(models.Manager):
def get_annual_commit_data_for_library(self, library, branch="master"):
"""Get the numbers of commits per year to a library and a branch."""
return (
self.filter(library=library, branch=branch)
.annotate(year=ExtractYear("month_year"))
.values("year")
.annotate(commit_count=Sum("commit_count"))
.order_by("year")
)
def get_commit_data_for_last_12_months_for_library(self, library, branch="master"):
"""Get the number of commits per month for the last 12 months to a library
and a branch."""
today = datetime.date.today()
one_year_ago = today - relativedelta(years=1)
return (
self.filter(
library=library,
month_year__range=(one_year_ago, today),
branch=branch,
)
.values("month_year")
.annotate(commit_count=Sum("commit_count"))
.order_by("month_year")
)

92
libraries/migrations/0021_commitauthor_commitauthoremail_commit_and_more.py Normal file
View File

@@ -0,0 +1,92 @@
# Generated by Django 4.2.15 on 2024-09-19 17:35
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("libraries", "0020_repopulate_slug_20240325_1616"),
]
operations = [
migrations.CreateModel(
name="CommitAuthor",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=100)),
("avatar_url", models.URLField(max_length=100, null=True)),
("github_profile_url", models.URLField(max_length=100, null=True)),
],
),
migrations.CreateModel(
name="CommitAuthorEmail",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("email", models.CharField(unique=True)),
(
"author",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="libraries.commitauthor",
),
),
],
),
migrations.CreateModel(
name="Commit",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("sha", models.CharField(max_length=40)),
("message", models.TextField(default="")),
("committed_at", models.DateTimeField(db_index=True)),
("is_merge", models.BooleanField(default=False)),
(
"author",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="libraries.commitauthor",
),
),
(
"library_version",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="libraries.libraryversion",
),
),
],
),
migrations.AddConstraint(
model_name="commit",
constraint=models.UniqueConstraint(
fields=("sha", "library_version"),
name="libraries_commit_sha_library_version_unique",
),
),
]

16
libraries/migrations/0022_delete_commitdata.py Normal file
View File

@@ -0,0 +1,16 @@
# Generated by Django 4.2.15 on 2024-09-23 18:42
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("libraries", "0021_commitauthor_commitauthoremail_commit_and_more"),
]
operations = [
migrations.DeleteModel(
name="CommitData",
),
]

83
libraries/models.py
View File

@@ -1,8 +1,9 @@
import re
from typing import Self
from urllib.parse import urlparse
from django.core.cache import caches
from django.db import models
from django.db import models, transaction
from django.utils.functional import cached_property
from django.utils.text import slugify
@@ -11,7 +12,6 @@ from core.markdown import process_md
from core.models import RenderedContent
from core.tasks import adoc_to_html
from .managers import CommitDataManager
from .utils import generate_random_string, write_content_to_tempfile
@@ -38,37 +38,58 @@ class Category(models.Model):
return super(Category, self).save(*args, **kwargs)
class CommitData(models.Model):
library = models.ForeignKey(
"libraries.Library",
on_delete=models.CASCADE,
help_text="The Library to which these commits belong.",
related_name="commit_data",
)
commit_count = models.PositiveIntegerField(
default=0, help_text="The number of commits made during the month."
)
month_year = models.DateField(
help_text="The month and year when the commits were made. Day is always set to "
"the first of the month."
)
branch = models.CharField(
max_length=256,
default="master",
help_text="The GitHub branch to which these commits were made.",
)
objects = CommitDataManager()
class Meta:
unique_together = ("library", "month_year", "branch")
verbose_name_plural = "Commit Data"
class CommitAuthor(models.Model):
name = models.CharField(max_length=100)
avatar_url = models.URLField(null=True, max_length=100)
github_profile_url = models.URLField(null=True, max_length=100)
def __str__(self):
return (
f"{self.library.name} commits for "
f"{self.month_year:%B %Y} to {self.branch} branch: {self.commit_count}"
)
return self.name
@transaction.atomic
def merge_author(self, other: Self):
"""Update references to `other` to point to `self`.
Deletes `other` after updating references.
"""
if self.pk == other.pk:
return
Commit.objects.filter(author=other).update(author=self)
other.commitauthoremail_set.update(author=self)
if not self.avatar_url:
self.avatar_url = other.avatar_url
if not self.github_profile_url:
self.github_profile_url = other.github_profile_url
self.save(update_fields=["avatar_url", "github_profile_url"])
other.delete()
class CommitAuthorEmail(models.Model):
author = models.ForeignKey(CommitAuthor, on_delete=models.CASCADE)
email = models.CharField(unique=True)
def __str__(self):
return f"{self.author.name}: {self.email}"
class Commit(models.Model):
author = models.ForeignKey(CommitAuthor, on_delete=models.CASCADE)
library_version = models.ForeignKey("LibraryVersion", on_delete=models.CASCADE)
sha = models.CharField(max_length=40)
message = models.TextField(default="")
committed_at = models.DateTimeField(db_index=True)
is_merge = models.BooleanField(default=False)
class Meta:
constraints = [
models.UniqueConstraint(
fields=["sha", "library_version"],
name="%(app_label)s_%(class)s_sha_library_version_unique",
)
]
def __str__(self):
return self.sha
class Library(models.Model):

26
libraries/tasks.py
View File

@@ -1,14 +1,12 @@
import structlog
from dateutil.relativedelta import relativedelta
from config.celery import app
from django.conf import settings
from django.db.models import Q
from django.utils import timezone
from core.boostrenderer import get_content_from_s3
from core.htmlhelper import get_library_documentation_urls
from libraries.github import LibraryUpdater
from libraries.models import LibraryVersion
from libraries.models import Library, LibraryVersion
from versions.models import Version
from .constants import (
LIBRARY_DOCS_EXCEPTIONS,
@@ -186,23 +184,15 @@ def update_libraries():
@app.task
def update_commit_counts(token=None):
"""Imports commit counts for all libraries, broken down by month, and saves
them to the database. See LibraryUpdater class for defaults.
"""
def update_commits(token=None):
updater = LibraryUpdater(token=token)
updater.update_monthly_commit_counts()
logger.info("libraries_update_commit_counts_finished")
for library in Library.objects.all():
updater.update_commits(obj=library)
logger.info("update_commits finished.")
@app.task
def update_current_month_commit_counts(token=None):
"""Imports commit counts for all libraries for the current month."""
def update_commit_author_github_data(token=None, clean=False):
updater = LibraryUpdater(token=token)
now = timezone.now()
# First of this month
since = timezone.make_aware(
timezone.datetime(year=now.year, month=now.month, day=1)
) - relativedelta(days=1)
updater.update_monthly_commit_counts(since=since, until=now)
logger.info("libraries_update_current_month_commit_counts_finished")
updater.update_commit_author_github_data(overwrite=clean)
logger.info("update_commit_author_github_data finished.")

9
libraries/tests/fixtures.py
View File

@@ -8,15 +8,6 @@ def category(db):
return baker.make("libraries.Category", name="Math", slug="math")
@pytest.fixture
def commit_data(library):
return baker.make(
"libraries.CommitData",
library=library,
commit_count=1,
)
@pytest.fixture
def library(db):
return baker.make(

43
libraries/tests/test_managers.py
View File

@@ -1,43 +0,0 @@
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from model_bakery import baker
from ..models import CommitData
def test_get_annual_commit_data_for_library(library):
five_years_ago = datetime.now().date().replace(year=datetime.now().year - 5)
for i in range(5):
date = five_years_ago.replace(year=five_years_ago.year + i)
baker.make(
"libraries.CommitData",
library=library,
month_year=date,
commit_count=i + 1,
branch="master",
)
result = CommitData.objects.get_annual_commit_data_for_library(library)
assert len(result) == 5
for i, data in enumerate(result):
assert data["year"] == five_years_ago.year + i
assert data["commit_count"] == i + 1
def test_get_commit_data_for_last_12_months_for_library(library):
one_year_ago = datetime.now().date() - timedelta(days=365)
for i in range(12):
date = one_year_ago + relativedelta(months=i)
baker.make(
"libraries.CommitData",
library=library,
month_year=date,
commit_count=i + 1,
branch="master",
)
result = CommitData.objects.get_commit_data_for_last_12_months_for_library(library)
assert len(result) == 12
for i, data in enumerate(result):
assert data["month_year"] == one_year_ago + relativedelta(months=i)
assert data["commit_count"] == i + 1

4
libraries/tests/test_models.py
View File

@@ -36,10 +36,6 @@ def test_category_creation(category):
assert category.name is not None
def test_commit_data_creation(commit_data):
assert commit_data.commit_count > 0
def test_library_creation(library):
assert library.versions.count() == 0

84
libraries/tests/test_views.py
View File

@@ -1,8 +1,6 @@
import datetime
import pytest
from django.utils import timezone
from dateutil.relativedelta import relativedelta
from model_bakery import baker
@@ -165,85 +163,27 @@ def test_library_docs_redirect(tp, library, library_version):
tp.response_302(resp)
def test_library_detail_context_get_commit_data_annual(tp, library_version):
def test_library_detail_context_get_commit_data_(tp, library_version):
"""
GET /libraries/{slug}/
Test that the method correctly retrieves the commit data
"""
library = library_version.library
random_library = baker.make("libraries.Library", slug="random")
current_year = timezone.now().year
for i in range(10):
year = current_year - i
date = datetime.date(year, 1, 1)
# Valid data
baker.make(
"libraries.CommitData",
library=library,
month_year=date,
commit_count=i + 1,
branch="master",
)
# Wrong library
baker.make(
"libraries.CommitData",
library=random_library,
month_year=date,
commit_count=i + 1,
branch="master",
)
# Wrong branch
baker.make(
"libraries.CommitData",
library=library,
month_year=date,
commit_count=i + 1,
branch="wrong-branch",
)
url = tp.reverse("library-detail", library.slug)
response = tp.get_check_200(url)
assert "commit_data_annual" in response.context
commit_data_annual = response.context["commit_data_annual"]
# Verify that the data is only for last 12 months for this library
assert len(commit_data_annual) == 10
for i, data in enumerate(reversed(commit_data_annual)):
assert data["date"] == datetime.date(current_year - i, 1, 1)
assert data["commit_count"] == i + 1
def test_library_detail_context_get_commit_data_last_12_months(tp, library_version):
"""
GET /libraries/{slug}/
Test that the commit_data_last_12_months var appears as expected
Test that the commit_data_by_release var appears as expected
"""
library = library_version.library
# Create CommitData for the library and another random library
random_library = baker.make("libraries.Library", slug="random")
version_a = baker.make("versions.Version", name="a")
version_b = baker.make("versions.Version", name="b")
version_c = baker.make("versions.Version", name="c")
current_month = timezone.now().date().replace(day=1)
for i in range(12):
date = current_month - relativedelta(months=i)
baker.make(
"libraries.CommitData", library=library, month_year=date, commit_count=i + 1
)
baker.make(
"libraries.CommitData",
library=random_library,
month_year=date,
commit_count=i + 1,
)
lv_a = baker.make("libraries.LibraryVersion", version=version_a, library=library)
lv_b = baker.make("libraries.LibraryVersion", version=version_b, library=library)
lv_c = baker.make("libraries.LibraryVersion", version=version_c, library=library)
for lv in [lv_a, lv_b, lv_c]:
baker.make("libraries.Commit", library_version=lv)
url = tp.reverse("library-detail", library.slug)
response = tp.get_check_200(url)
assert "commit_data_last_12_months" in response.context
commit_data_last_12_months = response.context["commit_data_last_12_months"]
# Verify that the data is only for last 12 months for this library
assert len(commit_data_last_12_months) == 12
for i, data in enumerate(reversed(commit_data_last_12_months)):
assert data["date"] == current_month - relativedelta(months=i)
assert data["commit_count"] == i + 1
assert "commit_data_by_release" in response.context
def test_library_detail_context_get_maintainers(tp, user, library_version):

155
libraries/views.py
View File

@@ -1,10 +1,11 @@
import datetime
from itertools import chain
import re
import structlog
from dateutil.relativedelta import relativedelta
from django.contrib import messages
from django.db.models import Count
from django.db.models import F, Count
from django.db.models.functions import Lower
from django.http import Http404
from django.shortcuts import get_object_or_404, redirect
from django.utils.decorators import method_decorator
@@ -17,7 +18,13 @@ from versions.models import Version
from .forms import VersionSelectionForm
from .mixins import VersionAlertMixin
from .models import Category, CommitData, Library, LibraryVersion
from .models import (
Category,
CommitAuthor,
CommitAuthorEmail,
Library,
LibraryVersion,
)
from .utils import (
redirect_to_view_with_params,
get_view_from_cookie,
@@ -257,10 +264,44 @@ class LibraryDetail(FormMixin, DetailView):
context["github_url"] = self.get_github_url(context["version"])
context["maintainers"] = self.get_maintainers(context["version"])
context["author_tag"] = self.get_author_tag()
exclude_maintainer_ids = [
getattr(x.commitauthor, "id")
for x in context["maintainers"]
if x.commitauthor
]
context["top_contributors_release"] = self.get_top_contributors(
version=context["version"],
exclude=exclude_maintainer_ids,
)
exclude_top_contributor_ids = [
x.id for x in context["top_contributors_release"]
]
context["top_contributors_overall"] = self.get_top_contributors(
exclude=exclude_maintainer_ids + exclude_top_contributor_ids
)
# Since we need to execute these queries separately anyway, just concatenate
# their results instead of making a new query
all_contributors = []
for x in chain(
context["top_contributors_release"], context["top_contributors_overall"]
):
all_contributors.append(
{
"name": x.name,
}
)
for x in context["maintainers"]:
all_contributors.append(
{
"name": x.get_full_name(),
}
)
all_contributors.sort(key=lambda x: x["name"].lower())
context["all_contributors"] = all_contributors
# Populate the commit graphs
context["commit_data_annual"] = self.get_commit_data_annual()
context["commit_data_last_12_months"] = self.get_commit_data_last_12_months()
context["commit_data_by_release"] = self.get_commit_data_by_release()
# Populate the library description
client = GithubAPIClient(repo_slug=self.object.github_repo)
@@ -270,6 +311,20 @@ class LibraryDetail(FormMixin, DetailView):
return context
def get_commit_data_by_release(self):
qs = (
LibraryVersion.objects.filter(library=self.object)
.annotate(count=Count("commit"), version_name=F("version__name"))
.order_by("-version__name")
)[:20]
return [
{
"release": x.version_name.strip("boost-"),
"commit_count": x.count,
}
for x in reversed(list(qs))
]
def get_object(self):
"""Get the current library object from the slug in the URL.
If present, use the version_slug to get the right LibraryVersion of the library.
@@ -313,60 +368,6 @@ class LibraryDetail(FormMixin, DetailView):
return commit_data_list
def get_commit_data_annual(self):
"""Retrieve number of commits to the library per year."""
if not self.object.commit_data.exists():
return []
# Get the first and last commit dates to determine the range of years
first_commit = self.object.commit_data.earliest("month_year")
first_year = first_commit.month_year.year
current_year = datetime.date.today().year
years = list(range(first_year, current_year + 1))
# For years there were no commits, return the year and the 0 count
commit_data_annual = {year: 0 for year in years}
actual_data = dict(
CommitData.objects.get_annual_commit_data_for_library(
self.object
).values_list("year", "commit_count")
)
commit_data_annual.update(actual_data)
prepared_commit_data = [
{"date": year, "commit_count": count}
for year, count in commit_data_annual.items()
]
# Sort the data by date
prepared_commit_data.sort(key=lambda x: x["date"])
return self._prepare_commit_data(prepared_commit_data, "annual")
def get_commit_data_last_12_months(self):
"""Retrieve the number of commits per month for the last year."""
if not self.object.commit_data.exists():
return []
# Generate default dict of last 12 months with 0 commits so we still see
# months with no commits
today = datetime.date.today()
months = [(today - relativedelta(months=i)).replace(day=1) for i in range(12)]
commit_data_monthly = {month: 0 for month in months}
# Update dict with real data from the database.
actual_data = dict(
CommitData.objects.get_commit_data_for_last_12_months_for_library(
self.object
).values_list("month_year", "commit_count")
)
commit_data_monthly.update(actual_data)
prepared_commit_data = [
{"date": month, "commit_count": count}
for month, count in commit_data_monthly.items()
]
# Sort the data by date
prepared_commit_data.sort(key=lambda x: x["date"])
result = self._prepare_commit_data(prepared_commit_data, "monthly")
return result
def get_current_library_version(self, version):
"""Return the library-version for the latest version of Boost"""
# Avoid raising an error if the library has been removed from the latest version
@@ -411,10 +412,42 @@ class LibraryDetail(FormMixin, DetailView):
return self.object.github_url
def get_maintainers(self, version):
"""Get the maintainers for the current LibraryVersion."""
"""Get the maintainers for the current LibraryVersion.
Also patches the CommitAuthor onto the user, if a matching email exists.
"""
obj = self.get_object()
library_version = LibraryVersion.objects.get(library=obj, version=version)
return library_version.maintainers.all()
qs = list(library_version.maintainers.all())
commit_authors = {
author_email.email: author_email
for author_email in CommitAuthorEmail.objects.annotate(
email_lower=Lower("email")
)
.filter(email_lower__in=[x.email.lower() for x in qs])
.select_related("author")
}
for user in qs:
if author_email := commit_authors.get(user.email.lower(), None):
user.commitauthor = author_email.author
else:
user.commitauthor = None
return qs
def get_top_contributors(self, version=None, exclude=None):
if version:
library_version = LibraryVersion.objects.get(
library=self.object, version=version
)
qs = CommitAuthor.objects.filter(commit__library_version=library_version)
else:
qs = CommitAuthor.objects.filter(
commit__library_version__library=self.object
)
if exclude:
qs = qs.exclude(id__in=exclude)
qs = qs.annotate(count=Count("commit")).order_by("-count")
return qs
def get_version(self):
"""Get the version of Boost for the library we're currently looking at."""

11
templates/admin/commit_author_change_list.html Normal file
View File

@@ -0,0 +1,11 @@
{% extends "admin/change_list.html" %}
{% load i18n admin_urls %}
{% block object-tools %}
<ul class="object-tools">
{% block object-tools-items %}
{{ block.super }}
<li><a href="{% url 'admin:commit_author_update_github_data' %}" class="addlink">{% trans "Update Github Avatar and URL" %}</a></li>
{% endblock %}
</ul>
{% endblock %}

11
templates/admin/commit_change_list.html Normal file
View File

@@ -0,0 +1,11 @@
{% extends "admin/change_list.html" %}
{% load i18n admin_urls %}
{% block object-tools %}
<ul class="object-tools">
{% block object-tools-items %}
{{ block.super }}
<li><a href="{% url 'admin:update_commits' %}" class="addlink">{% trans "Update Commits" %}</a></li>
{% endblock %}
</ul>
{% endblock %}

1
templates/admin/library_change_list.html
View File

@@ -6,6 +6,7 @@
{% block object-tools-items %}
{{ block.super }}
<li><a href="{% url 'admin:update_libraries' %}" class="addlink">{% trans "Update Library Data" %}</a></li>
<li><a href="{% url 'admin:library_report_form' %}" class="addlink">{% trans "Get Release Report" %}</a></li>
{% endblock %}
</ul>
{% endblock %}

199
templates/admin/library_report_detail.html Normal file
View File

@@ -0,0 +1,199 @@
{% load static humanize %}
<!DOCTYPE html>
<html>
<head>
<title>
{% block title %}Boost{% endblock %}
</title>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="description" content="{% block description %}{% endblock %}" />
<meta name="keywords" content="{% block keywords %}{% endblock %}" />
<meta name="author"
content="{% block author %}Boost C++ Libraries{% endblock %}" />
<link rel="shortcut icon"
href="{% static 'img/Boost_Symbol_Transparent.svg' %}"
type="image/x-icon" />
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Noto+Sans:ital,wght@0,100..900;1,100..900&display=swap"
rel="stylesheet">
<link href="https://fonts.googleapis.com/css2?family=Noto+Sans+Mono&display=swap"
rel="stylesheet">
<link rel="stylesheet"
href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@24,400,0,0" />
<link rel="stylesheet"
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/all.css" />
<!-- TODO bring this local if we like it -->
<script src="https://cdn.jsdelivr.net/npm/apexcharts"></script>
<script src="{% static 'js/boost-gecko/main.062e4862.js' %}" defer></script>
{% block extra_head %}
<link href="{% static 'css/styles.css' %}" rel="stylesheet" />
{% endblock %}
{% block css %}
{% endblock css %}
<style>
@page {
margin: 0;
size: 300mm 180mm;
}
.pdf-page {
padding: 5mm;
height: 180mm;
width: 300mm;
page-break-after: always;
}
</style>
</head>
{% with bg_color='bg-gradient-to-tr from-[#7ac3e6]/50 to-[#d9b05e]/50' %}
<body>
<div>
<div class="pdf-page grid grid-cols-2 gap-x-4 items-center justify-items-center {{ bg_color }}">
<div>
<h1>Boost</h1>
<div>{{ commit_count|intcomma }} commits up through {{ version.display_name }}</div>
</div>
<div>There were {{ version_commit_count|intcomma }} commits in {{ version.display_name }}</div>
</div>
<div class="pdf-page grid grid-cols-2 gap-x-4 items-center justify-items-center {{ bg_color }}">
<div class="flex flex-col">
<h1 class="mx-auto">Boost {{ version.display_name }}</h1>
<div class="mx-auto mb-4">{{ version_commit_count|intcomma }} Commits Across {{ library_count }} Repositories</div>
<div class="flex gap-x-2">
<div>
<div class="grid grid-cols-5 gap-2">
{% for author in top_contributors_release_overall %}
<div class="flex flex-col gap-y-2 w-20">
{% if author.avatar_url %}
<img src="{{ author.avatar_url }}"
alt="{{ author.name }}"
class="w-8 h-8 rounded mx-auto">
{% else %}
<div class="w-8 h-8 rounded bg-gray-300 mx-auto"></div>
{% endif %}
<div class="w-full flex flex-col">
<div class="text-[0.6rem] overflow-ellipsis overflow-hidden whitespace-nowrap w-full text-center">
{{ author.name }}
</div>
<div class="text-[0.6rem] mx-auto">({{ author.commit_count }})</div>
</div>
</div>
{% endfor %}
</div>
</div>
</div>
</div>
<div class="flex flex-col">
<h3 class="mx-auto">Most Committed Libraries</h3>
<div id="top-committed-libraries-chart" class="w-full text-center"></div>
</div>
</div>
{% for item in library_data %}
<div class="pdf-page grid grid-cols-2 gap-x-4 items-center justify-items-center {{ bg_color }}">
<div>
<h3>{{ item.library.name }}</h3>
<div>{{ item.library.description }}</div>
</div>
<div class="flex flex-col gap-y-8">
<h4>There were {{ item.version_count.commit_count }} commits in release {{ version.display_name }}</h4>
{% if item.new_contributors_count.count > 1 %}
<div>
There were {{ item.new_contributors_count.count }} new contributors this release!
</div>
{% elif item.new_contributors_count.count == 1 %}
<div>
There was {{ item.new_contributors_count.count }} new contributor this release!
</div>
{% endif %}
<div>
<div class="mb-2">Top Contributors</div>
<div class="grid grid-cols-5 gap-2 flex-wrap">
{% for author in item.top_contributors_release %}
<div class="flex flex-col gap-y-2 w-20">
{% if author.avatar_url %}
<img src="{{ author.avatar_url }}"
alt="{{ author.name }}"
class="w-8 h-8 rounded mx-auto">
{% else %}
<div class="w-8 h-8 rounded bg-gray-300 mx-auto"></div>
{% endif %}
<div class="w-full flex flex-col justify-center items-center">
<div class="text-[0.6rem] overflow-ellipsis overflow-hidden whitespace-nowrap w-full text-center">
{{ author.name }}
</div>
<div class="text-[0.6rem]">({{ author.commit_count }})</div>
</div>
</div>
{% endfor %}
</div>
</div>
</div>
</div>
{% endfor %}
</div>
<div class="pdf-page {{ bg_color }}" style="page-break-after: avoid;">This is the last page</div>
<script>
var options = {
series: [{
name: 'Commits',
data: [{% for library in top_libraries_for_version %}{{library.commit_count}}, {% endfor %}]
}],
chart: {
height: 300,
type: 'bar',
foreColor: '#373d3f',
background: '#ffffff00',
toolbar: {
show: false,
},
},
plotOptions: {
bar: {
borderRadius: 2,
dataLabels: {
position: 'top', // top, center, bottom
},
}
},
dataLabels: {
offsetY: -16,
enabled: true,
style: {
fontSize: '11px',
colors: ["rgb(49, 74, 87)"],
}
},
xaxis: {
categories: [{% for library in top_libraries_for_version %} "{{ library.name }}", {% endfor %}],
position: 'bottom',
axisBorder: {
show: false
},
axisTicks: {
show: false
},
tooltip: {
enabled: true,
}
},
yaxis: {
axisBorder: {
show: true
},
axisTicks: {
show: true,
},
labels: {
show: true,
}
},
};
// SS: putting this in the window object, a bit hacky, to be able to access it in the light/dark switcher - probably a better way
const chart = new ApexCharts(document.querySelector("#top-committed-libraries-chart"), options);
chart.render();
</script>
</body>
{% endwith %}
</html>

14
templates/admin/library_report_form.html Normal file
View File

@@ -0,0 +1,14 @@
{% extends "admin/base_site.html" %}
{% load static %}
{% block content %}
{{ block.super }}
<div class='container mx-auto'>
<h1>Generate Report for Release</h1>
<div>
<form action="">
{{ form.as_p }}
<input class="default" type="submit" />
</form>
</div>
</div>
{% endblock content %}

99
templates/admin/library_stat_detail.html Normal file
View File

@@ -0,0 +1,99 @@
{% extends "admin/base_site.html" %}
{% load static %}
{% block extrahead %}
<link href="{% static 'css/styles.css' %}" rel="stylesheet" />
{% endblock extrahead %}
{% block content %}
{{ block.super }}
<div class='container mx-auto'>
<h1>
{{ object.name }} Stats
</h1>
<div class="flex gap-x-4 flex-wrap">
<div>
<h3 class="mb-2">
Commit Count By Release
</h3>
<div class="flex flex-col gap-y-1">
{% for release in commits_per_release %}
<div>
{{ release.version_name }}: {{ release.count }}
</div>
{% endfor %}
<div>
</div>
</div>
</div>
<div>
<h3 class="mb-2">
Top Contributors Overall
</h3>
<div class="flex flex-col gap-y-1">
{% for author in commits_per_author %}
<div class="flex gap-x-1">
{% if author.avatar_url %}
<img src="{{ author.avatar_url }}" alt="github avatar" class="w-8 rounded">
{% else %}
<div class="w-8 h-8 rounded bg-silver">
</div>
{% endif %}
<div>
{{ author.name }}: {{ author.count }}
</div>
</div>
{% endfor %}
</div>
</div>
<div>
<h3 class="mb-2">
Top 3 Contributors Per Release
</h3>
<div class="flex flex-col gap-y-1">
{% for item in commits_per_author_release %}
{% ifchanged item.version__name %}
<h3 class="my-2">
{{ item.version__name }}
</h3>
<hr>
{% endifchanged %}
<div class="flex gap-x-1">
{% if item.commit__author__avatar_url %}
<img src="{{ item.commit__author__avatar_url }}"
alt="github avatar"
class="w-8 rounded">
{% else %}
<div class="w-8 h-8 rounded bg-silver">
</div>
{% endif %}
<div>
{{ item.commit__author__name }}: {{ item.count }}
</div>
</div>
{% endfor %}
</div>
</div>
<div>
<h3 class="mb-2">
New Contributors Per Release
</h3>
<div class="flex flex-col gap-y-1">
{% for lv in new_contributor_counts %}
<div class="grid grid-cols-2 gap-2">
<div>
{{ lv.version.display_name }}
</div>
<div>
{{ lv.count }} new contributors ({{ lv.up_to_count }} total)
</div>
</div>
{% endfor %}
</div>
</div>
</div>
</div>
{% endblock content %}

4
templates/base.html
View File

@@ -3,7 +3,9 @@
<html>
<head>
<script defer data-domain="preview.boost.org" src="https://plausible.io/js/script.js"></script>
<script defer
data-domain="preview.boost.org"
src="https://plausible.io/js/script.js"></script>
<title>{% block title %}Boost{% endblock %}</title>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />

101
templates/libraries/detail.html
View File

@@ -103,12 +103,8 @@
<div class="pb-2 px-0 ml-0 w-full h-auto md:w-2/3 md:pb-0 md:pl-6 md:mx-6">
<div class="flex">
<div class="flex w-full">
{# todo: add a toggle between the two types #}
<div class="w-full">
<div id="chart1" class="w-full text-center">{# Commits per Month #}</div>
</div>
<div class="w-full hidden">
<div id="chart2" class="w-full text-center">{# Commits per Year #}</div>
<div id="chart1" class="w-full text-center">{# Commits per Release #}</div>
</div>
</div>
</div>
@@ -116,26 +112,78 @@
</section>
{% if maintainers %}
{% if maintainers or top_contributors_release or top_contributors_overall %}
<section class="p-6 pt-1 my-4 bg-white md:rounded-lg md:shadow-lg dark:text-white text-slate dark:bg-charcoal dark:bg-neutral-700">
<!-- Avatars -->
<h2 class="text-2xl">Maintainers</h2>
<div class="flex flex-wrap justify-start">
{% for user in maintainers %}
<div class="p-1 md:p-2 w-min text-center flex flex-col items-center">
<div class="bg-gray-300 dark:bg-slate rounded-lg h-9 w-9 overflow-hidden">
{% if user.image %}
<img src="{{ user.image.url }}"
title="{{ user.get_full_name }}"
alt="{{ user.get_full_name }}"
class="rounded-lg h-9 w-9 object-cover" />
{% else %}
<i class="h-9 w-9 m-auto text-3xl fas fa-user text-white dark:text-white/60 " title="{{ user.get_full_name }}"></i>
{% endif %}
<h2 class="text-2xl">Maintainers &amp; Contributors</h2>
<div class="flex flex-col gap-y-4">
<div class="flex flex-wrap justify-center">
{% for user in maintainers %}
<a {% if user.commitauthor.github_profile_url %}href="{{ user.commitauthor.github_profile_url }}"{% endif %}>
<div class="p-1 md:p-2 w-min text-center flex flex-col items-center">
<div class="bg-gray-300 dark:bg-slate rounded-lg h-12 w-12">
{% if user.image or user.commitauthor.avatar_url %}
<img src="{% if user.image %}{{ user.image.url}}{% else %}{{ user.commitauthor.avatar_url }}{% endif %}"
title="{{ user.get_full_name }}"
alt="{{ user.get_full_name }}"
class="rounded-lg h-12 w-12 object-cover" />
{% else %}
<i class="h-12 w-12 m-auto text-3xl align-middle fas fa-user text-white dark:text-white/60 " title="{{ user.get_full_name }}"></i>
{% endif %}
</div>
<span class="text-xs">{{ user.get_full_name }}</span>
</div>
<span class="text-xs">{{ user.get_full_name }}</span>
</div>
{% endfor %}
</a>
{% endfor %}
</div>
<div class="flex flex-wrap justify-center">
{% for author in top_contributors_release %}
<a
{% if author.github_profile_url %}
href="{{ author.github_profile_url }}"
{% endif %}
>
<div class="p-1 md:p-2 flex text-center justify-center" title="{{ author.name }}">
<div class="bg-gray-300 dark:bg-slate rounded-lg h-9 w-9 overflow-hidden">
{% if author.avatar_url %}
<img src="{{ author.avatar_url }}"
title="{{ author.name }}"
alt="{{ author.name }}"
class="rounded-lg h-9 w-9 object-cover" />
{% else %}
<i class="h-9 w-9 m-auto text-3xl fas fa-user text-white dark:text-white/60 " title="{{ author.name }}"></i>
{% endif %}
</div>
</div>
</a>
{% endfor %}
</div>
<div class="flex flex-wrap justify-center">
{% for author in top_contributors_overall %}
<a {% if author.github_profile_url %}href="{{ author.github_profile_url }}"{% endif %}>
<div class="p-1 md:p-2 flex text-center justify-center" title="{{ author.name }}">
<div class="bg-gray-300 dark:bg-slate rounded-lg h-9 w-9 overflow-hidden">
{% if author.avatar_url %}
<img src="{{ author.avatar_url }}"
title="{{ author.name }}"
alt="{{ author.name }}"
class="rounded-lg h-9 w-9 object-cover" />
{% else %}
<i class="h-9 w-9 m-auto text-3xl fas fa-user text-white dark:text-white/60 " title="{{ author.name }}"></i>
{% endif %}
</div>
</div>
</a>
{% endfor %}
</div>
<div class="flex flex-wrap justify-center gap-x-4 gap-y-2">
{% for author in all_contributors %}
<div>{{ author.name }}</div>
{% endfor %}
</div>
</div>
</section>
{% endif %}
@@ -153,13 +201,16 @@
var options = {
series: [{
name: 'Commits',
data: [{% for commit_data in commit_data_last_12_months %}{{ commit_data.commit_count }}, {% endfor %}]
data: [{% for commit_data in commit_data_by_release %}{{ commit_data.commit_count }}, {% endfor %}]
}],
chart: {
height: 350,
type: 'bar',
foreColor: localStorage.getItem('colorMode') === 'dark' ? '#f6f7f8' : '#373d3f',
background: localStorage.getItem('colorMode') === 'dark' ? '#172A34' : '#ffffff',
toolbar: {
show: false,
},
},
theme: {
mode: localStorage.getItem('colorMode') === 'dark' ? 'dark' : 'light',
@@ -189,7 +240,7 @@
},
xaxis: {
categories: [{% for commit_data in commit_data_last_12_months %}"{{ commit_data.date|date:'M' }}",{% endfor %}],
categories: [{% for commit_data in commit_data_by_release %}"{{ commit_data.release }}",{% endfor %}],
position: 'top',
axisBorder: {
show: false
@@ -219,7 +270,7 @@
},
title: {
text: 'Commits per Month',
text: 'Commits Per Release',
floating: true,
offsetY: 330,
align: 'center',