Generate reports asynchronously (#1421)

- fixes #1415

libraries/admin.py

@@ -2,11 +2,12 @@ from django.contrib import admin
 from django.db import transaction
 from django.db.models import F, Count, OuterRef, Window
 from django.db.models.functions import RowNumber
-from django.http import HttpResponseRedirect
+from django.http import HttpResponse, HttpResponseRedirect
 from django.template.response import TemplateResponse
 from django.urls import path, reverse
 from django.utils.safestring import mark_safe
 from django.shortcuts import redirect
+from django.views.generic import TemplateView
 
 from libraries.forms import CreateReportForm, CreateReportFullForm
 from versions.models import Version
@@ -22,12 +23,14 @@ from .models import (
     PullRequest,
 )
 from .tasks import (
+    generate_library_report,
     update_authors_and_maintainers,
     update_commit_author_github_data,
     update_commits,
     update_issues,
     update_libraries,
     update_library_version_documentation_urls_all_versions,
+    generate_release_report,
 )
 
 
@@ -130,6 +133,72 @@ class LibraryVersionInline(admin.TabularInline):
     fields = ["version", "documentation_url"]
 
 
+class ReleaseReportView(TemplateView):
+    polling_template = "admin/report_polling.html"
+    form_template = "admin/library_report_form.html"
+    form_class = CreateReportForm
+    report_type = "release report"
+
+    def get_template_names(self):
+        if not self.request.GET.get("submit", None):
+            return [self.form_template]
+        form = self.get_form()
+        if not form.is_valid():
+            return [self.form_template]
+        if form.cleaned_data["no_cache"]:
+            return [self.form_template]
+        content = form.cache_get()
+        if content:
+            if not content.content_html:
+                return [self.polling_template]
+        else:
+            return [self.polling_template]
+
+    def get_form(self):
+        data = None
+        if self.request.GET.get("submit", None):
+            data = self.request.GET
+        return self.form_class(data)
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context["report_type"] = self.report_type
+        context["form"] = self.get_form()
+        return context
+
+    def generate_report(self):
+        generate_release_report.delay(self.request.GET)
+
+    def get(self, request, *args, **kwargs):
+        form = self.get_form()
+        if form.is_valid():
+            if form.cleaned_data["no_cache"]:
+                params = request.GET.copy()
+                form.cache_clear()
+                del params["no_cache"]
+                return redirect(request.path + f"?{params.urlencode()}")
+            content = form.cache_get()
+            if not content:
+                # Ensure a RenderedContent exists so the task is not re-queued
+                form.cache_set("")
+                self.generate_report()
+            elif content.content_html:
+                return HttpResponse(content.content_html)
+        return TemplateResponse(
+            request,
+            self.get_template_names(),
+            self.get_context_data(),
+        )
+
+
+class LibraryReportView(ReleaseReportView):
+    form_class = CreateReportFullForm
+    report_type = "library report"
+
+    def generate_report(self):
+        generate_library_report.delay(self.request.GET)
+
+
 @admin.register(Library)
 class LibraryAdmin(admin.ModelAdmin):
     list_display = ["name", "key", "github_url", "view_stats"]
@@ -153,77 +222,19 @@ class LibraryAdmin(admin.ModelAdmin):
                 self.admin_site.admin_view(self.library_stat_detail),
                 name="library_stat_detail",
             ),
-            path(
-                "release-report-form/",
-                self.admin_site.admin_view(self.release_report_form),
-                name="release_report_form",
-            ),
             path(
                 "release-report/",
-                self.admin_site.admin_view(self.release_report_view),
+                self.admin_site.admin_view(ReleaseReportView.as_view()),
                 name="release_report",
             ),
             path(
-                "report-full-form/",
-                self.admin_site.admin_view(self.report_form_full_view),
-                name="library_report_full_form",
-            ),
-            path(
-                "report-full/",
-                self.admin_site.admin_view(self.report_full_view),
+                "library-report/",
+                self.admin_site.admin_view(LibraryReportView.as_view()),
                 name="library_report_full",
             ),
         ]
         return my_urls + urls
 
-    def release_report_form(self, request):
-        form = CreateReportForm()
-        context = {}
-        if request.GET.get("submit", None):
-            form = CreateReportForm(request.GET)
-            if form.is_valid():
-                context.update(form.get_stats())
-                return redirect(
-                    reverse("admin:release_report") + f"?{request.GET.urlencode()}"
-                )
-        if not context:
-            context["form"] = form
-        return TemplateResponse(request, "admin/library_report_form.html", context)
-
-    def release_report_view(self, request):
-        form = CreateReportForm(request.GET)
-        context = {"form": form}
-        if form.is_valid():
-            context.update(form.get_stats())
-        else:
-            return redirect("admin:release_report_form")
-        return TemplateResponse(request, "admin/release_report_detail.html", context)
-
-    def report_form_full_view(self, request):
-        form = CreateReportFullForm()
-        context = {}
-        if request.GET.get("submit", None):
-            form = CreateReportFullForm(request.GET)
-            if form.is_valid():
-                context.update(form.get_stats())
-                return redirect(
-                    reverse("admin:library_report_full") + f"?{request.GET.urlencode()}"
-                )
-        if not context:
-            context["form"] = form
-        return TemplateResponse(request, "admin/library_report_form.html", context)
-
-    def report_full_view(self, request):
-        form = CreateReportFullForm(request.GET)
-        context = {"form": form}
-        if form.is_valid():
-            context.update(form.get_stats())
-        else:
-            return redirect("admin:library_report_full_form")
-        return TemplateResponse(
-            request, "admin/library_report_full_detail.html", context
-        )
-
     def view_stats(self, instance):
         url = reverse("admin:library_stat_detail", kwargs={"pk": instance.pk})
         return mark_safe(f"<a href='{url}'>View Stats</a>")

libraries/forms.py

@@ -5,10 +5,12 @@ import psycopg2
 from wordcloud import WordCloud, STOPWORDS
 from matplotlib import pyplot as plt
 
+from django.template.loader import render_to_string
 from django.db.models import F, Q, Count, OuterRef, Sum
-from django.forms import Form, ModelChoiceField, ModelForm
+from django.forms import Form, ModelChoiceField, ModelForm, BooleanField
 from django.conf import settings
 
+from core.models import RenderedContent
 from versions.models import Version
 from .models import Commit, CommitAuthor, Issue, Library, LibraryVersion
 from mailing_list.models import EmailData
@@ -34,6 +36,8 @@ class VersionSelectionForm(Form):
 class CreateReportFullForm(Form):
     """Form for creating a report over all releases."""
 
+    html_template_name = "admin/library_report_full_detail.html"
+
     library_queryset = Library.objects.all().order_by("name")
     library_1 = ModelChoiceField(
         queryset=library_queryset,
@@ -68,6 +72,26 @@ class CreateReportFullForm(Form):
         queryset=library_queryset,
         required=False,
     )
+    no_cache = BooleanField(
+        required=False,
+        initial=False,
+        help_text="Force the page to be regenerated, do not use cache.",
+    )
+
+    @property
+    def cache_key(self):
+        chosen_libraries = [
+            self.cleaned_data["library_1"],
+            self.cleaned_data["library_2"],
+            self.cleaned_data["library_3"],
+            self.cleaned_data["library_4"],
+            self.cleaned_data["library_5"],
+            self.cleaned_data["library_6"],
+            self.cleaned_data["library_7"],
+            self.cleaned_data["library_8"],
+        ]
+        lib_string = ",".join(str(x.id) if x else "" for x in chosen_libraries)
+        return f"full-report-{lib_string}"
 
     def _get_top_libraries(self):
         return (
@@ -168,10 +192,37 @@ class CreateReportFullForm(Form):
             "library_count": Library.objects.all().count(),
         }
 
+    def cache_html(self):
+        """Render and cache the html for this report."""
+        # ensure we have "cleaned_data"
+        if not self.is_valid():
+            return ""
+        html = render_to_string(self.html_template_name, self.get_stats())
+        self.cache_set(html)
+        return html
+
+    def cache_get(self) -> RenderedContent | None:
+        return RenderedContent.objects.filter(cache_key=self.cache_key).first()
+
+    def cache_clear(self):
+        return RenderedContent.objects.filter(cache_key=self.cache_key).delete()
+
+    def cache_set(self, content_html):
+        """Cache the html for this report."""
+        return RenderedContent.objects.update_or_create(
+            cache_key=self.cache_key,
+            defaults={
+                "content_html": content_html,
+                "content_type": "text/html",
+            },
+        )
+
 
 class CreateReportForm(CreateReportFullForm):
     """Form for creating a report for a specific release."""
 
+    html_template_name = "admin/release_report_detail.html"
+
     version = ModelChoiceField(
         queryset=Version.objects.minor_versions().order_by("-version_array")
     )
@@ -182,6 +233,22 @@ class CreateReportForm(CreateReportFullForm):
             "library_1"
         ].help_text = "If none are selected, all libraries will be selected."
 
+    @property
+    def cache_key(self):
+        chosen_libraries = [
+            self.cleaned_data["library_1"],
+            self.cleaned_data["library_2"],
+            self.cleaned_data["library_3"],
+            self.cleaned_data["library_4"],
+            self.cleaned_data["library_5"],
+            self.cleaned_data["library_6"],
+            self.cleaned_data["library_7"],
+            self.cleaned_data["library_8"],
+        ]
+        lib_string = ",".join(str(x.id) if x else "" for x in chosen_libraries)
+        version = self.cleaned_data["version"]
+        return f"release-report-{lib_string}-{version.name}"
+
     def _get_top_contributors_for_version(self):
         return (
             CommitAuthor.objects.filter(

libraries/tasks.py

@@ -6,6 +6,7 @@ from django.conf import settings
 from django.db.models import Q
 from core.boostrenderer import get_content_from_s3
 from core.htmlhelper import get_library_documentation_urls
+from libraries.forms import CreateReportForm, CreateReportFullForm
 from libraries.github import LibraryUpdater
 from libraries.models import Library, LibraryVersion
 from versions.models import Version
@@ -212,3 +213,17 @@ def update_issues(clean=False):
     if clean:
         command.append("--clean")
     call_command(*command)
+
+
+@app.task
+def generate_release_report(params):
+    """Generate a release report asynchronously and save it in RenderedContent."""
+    form = CreateReportForm(params)
+    form.cache_html()
+
+
+@app.task
+def generate_library_report(params):
+    """Generate a library report asynchronously and save it in RenderedContent."""
+    form = CreateReportFullForm(params)
+    form.cache_html()

templates/admin/library_change_list.html

@@ -7,8 +7,8 @@
         {{ block.super }}
         <li><a href="{% url 'admin:update_libraries' %}" class="addlink">{% trans "Update Library Data" %}</a></li>
         <li><a href="{% url 'admin:update_authors_and_maintainers' %}" class="addlink">{% trans "Update Authors & Maintainers" %}</a></li>
-        <li><a href="{% url 'admin:release_report_form' %}" class="addlink">{% trans "Get Release Report" %}</a></li>
-        <li><a href="{% url 'admin:library_report_full_form' %}" class="addlink">{% trans "Get Library Report" %}</a></li>
+        <li><a href="{% url 'admin:release_report' %}" class="addlink">{% trans "Get Release Report" %}</a></li>
+        <li><a href="{% url 'admin:library_report_full' %}" class="addlink">{% trans "Get Library Report" %}</a></li>
     {% endblock %}
 </ul>
 {% endblock %}

templates/admin/report_polling.html

@@ -0,0 +1,15 @@
+{% extends "admin/library_report_base.html" %}
+
+{% block content %}
+  <div class="flex container my-4 mx-auto">
+    The {{ report_type }} is being generated. This page will refresh periodically, please wait.
+  </div>
+
+  <script>
+    document.addEventListener("DOMContentLoaded", function() {
+      setTimeout(function () {
+        window.location.reload()
+      }, 10000);
+    })
+  </script>
+{% endblock content %}