diff --git a/config/settings.py b/config/settings.py index 0dde0697..b58eee49 100755 --- a/config/settings.py +++ b/config/settings.py @@ -285,6 +285,8 @@ CACHES = { }, } +ENABLE_DB_CACHE = env.bool("ENABLE_DB_CACHE", default=False) + # Default interval by which to clear the static content cache CLEAR_STATIC_CONTENT_CACHE_DAYS = 7 diff --git a/core/admin.py b/core/admin.py index a7229acf..8da8c37f 100644 --- a/core/admin.py +++ b/core/admin.py @@ -4,7 +4,7 @@ from .models import RenderedContent, SiteSettings @admin.register(RenderedContent) class RenderedContentAdmin(admin.ModelAdmin): - list_display = ("cache_key", "content_type") + list_display = ("cache_key", "content_type", "modified") search_fields = ("cache_key",) diff --git a/core/tasks.py b/core/tasks.py index 667863c0..9431b053 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -84,4 +84,3 @@ def save_rendered_content(cache_key, content_type, content_html, last_updated_at obj_id=obj.id, obj_created=created, ) - return obj diff --git a/core/views.py b/core/views.py index d256183f..6a5fe9d2 100644 --- a/core/views.py +++ b/core/views.py @@ -21,6 +21,7 @@ from django.views import View from django.views.generic import TemplateView from requests.compat import chardet +from config.settings import ENABLE_DB_CACHE from libraries.constants import LATEST_RELEASE_URL_PATH_STR from libraries.utils import legacy_path_transform from versions.models import Version @@ -216,6 +217,7 @@ class ContentNotFoundException(Exception): class BaseStaticContentTemplateView(TemplateView): template_name = "adoc_content.html" + allowed_db_save_types = {"text/asciidoc"} def get(self, request, *args, **kwargs): """Return static content that originates in S3. @@ -380,11 +382,9 @@ class BaseStaticContentTemplateView(TemplateView): return HttpResponse(content, content_type=context["content_type"]) def save_to_database(self, cache_key, result): - """Saves the rendered asciidoc content to the database via celery.""" + """Saves the rendered asciidoc content to the database.""" content_type = result.get("content_type") - last_updated_at_raw = result.get("last_updated_at") - - if content_type == "text/asciidoc": + if content_type in self.allowed_db_save_types: last_updated_at_raw = result.get("last_updated_at") last_updated_at = ( parse(last_updated_at_raw) if last_updated_at_raw else None @@ -510,6 +510,13 @@ FULLY_MODERNIZED_LIB_VERSIONS = [ class DocLibsTemplateView(BaseStaticContentTemplateView): + allowed_db_save_types = { + "text/asciidoc", + "text/html", + "text/html; charset=utf-8", + "text/css; charset=utf-8", + } + def get_from_s3(self, content_path): legacy_url = normalize_boost_doc_path(content_path) return super().get_from_s3(legacy_url) @@ -544,6 +551,30 @@ class DocLibsTemplateView(BaseStaticContentTemplateView): return render_to_string("original_docs.html", context, request=self.request) + def get_content(self, content_path): + """Return content from database (cache) or S3.""" + # For now at least we're only going to cache docs this way, user guides and + # will continue to be cached as they were + + if not ENABLE_DB_CACHE: + return self.get_from_s3(content_path) + + cache_key = f"static_content_{content_path}" + # check to see if in db, if not retrieve from s3 and save to db + result = self.get_from_database(cache_key) + if not result and (result := self.get_from_s3(content_path)): + self.save_to_database(cache_key, result) + + if result is None: + logger.info( + "get_content_from_s3_view_no_valid_object", + key=content_path, + status_code=404, + ) + raise ContentNotFoundException("Content not found") + + return result + def _fully_modernize_content(self, content): """For libraries that have opted in to boostlook modernization""" content_type = self.content_dict.get("content_type") diff --git a/env.template b/env.template index 4e39a5ab..936eb0d0 100644 --- a/env.template +++ b/env.template @@ -42,6 +42,9 @@ MAILMAN_ADMIN_USER="" MAILMAN_ADMIN_EMAIL="" SERVE_FROM_DOMAIN=localhost +# Postgres caching of pages, currently only used for docs +ENABLE_DB_CACHE=True + # Celery settings CELERY_BROKER=redis://redis:6379/0 CELERY_BACKEND=redis://redis:6379/0 diff --git a/kube/boost/values-cppal-dev-gke.yaml b/kube/boost/values-cppal-dev-gke.yaml index 81ba40c5..d75038a5 100644 --- a/kube/boost/values-cppal-dev-gke.yaml +++ b/kube/boost/values-cppal-dev-gke.yaml @@ -191,6 +191,9 @@ Env: value: redis://redis:6379/0 - name: CELERY_BACKEND value: redis://redis:6379/0 + # postgres caching of s3 text file content + - name: ENABLE_DB_CACHE + value: "true" # Volumes Volumes: diff --git a/kube/boost/values-production-gke.yaml b/kube/boost/values-production-gke.yaml index 61aa7dbd..f1e262a6 100644 --- a/kube/boost/values-production-gke.yaml +++ b/kube/boost/values-production-gke.yaml @@ -191,6 +191,9 @@ Env: value: redis://redis:6379/0 - name: CELERY_BACKEND value: redis://redis:6379/0 + # postgres caching of s3 text file content + - name: ENABLE_DB_CACHE + value: "true" # Volumes Volumes: diff --git a/kube/boost/values-stage-gke.yaml b/kube/boost/values-stage-gke.yaml index 391f7bd8..f27b53bd 100644 --- a/kube/boost/values-stage-gke.yaml +++ b/kube/boost/values-stage-gke.yaml @@ -191,6 +191,9 @@ Env: value: redis://redis:6379/0 - name: CELERY_BACKEND value: redis://redis:6379/0 + # postgres caching of s3 text file content + - name: ENABLE_DB_CACHE + value: "true" # Volumes Volumes: diff --git a/kube/boost/values.yaml b/kube/boost/values.yaml index 23c8a172..cea4eed2 100644 --- a/kube/boost/values.yaml +++ b/kube/boost/values.yaml @@ -167,6 +167,9 @@ Env: # Static content cache timeout - name: STATIC_CACHE_TIMEOUT value: "60" + # postgres caching of s3 text file content + - name: ENABLE_DB_CACHE + value: "true" # Volumes Volumes: