mirror of
https://github.com/boostorg/website-v2.git
synced 2026-01-19 04:42:17 +00:00
Add task to save rendered content to db; call tasks on delay
This commit is contained in:
Lacey Williams Henschel
@@ -5,7 +5,7 @@ import tempfile
|
||||
from .boostrenderer import get_body_from_html
|
||||
|
||||
|
||||
def convert_adoc_to_html(file_path, delete_file= True):
|
||||
def convert_adoc_to_html(file_path, delete_file=True):
|
||||
"""
|
||||
Converts an AsciiDoc file to HTML.
|
||||
If delete_file is True, the temporary file will be deleted after the
|
||||
|
||||
@@ -1,42 +1,83 @@
|
||||
import os
|
||||
|
||||
import subprocess
|
||||
import structlog
|
||||
|
||||
from celery import shared_task
|
||||
|
||||
from .asciidoc import adoc_to_html
|
||||
from .boostrenderer import get_body_from_html, get_content_from_s3
|
||||
from dateutil.parser import parse
|
||||
|
||||
from django.core.cache import caches
|
||||
|
||||
from .asciidoc import convert_adoc_to_html, process_adoc_to_html_content
|
||||
from .boostrenderer import get_content_from_s3
|
||||
from .models import RenderedContent
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
@shared_task
|
||||
def adoc_to_html(file_path, delete_file=True):
|
||||
return adoc_to_html(file_path, delete_file=delete_file)
|
||||
return convert_adoc_to_html(file_path, delete_file=delete_file)
|
||||
|
||||
|
||||
@shared_task
|
||||
def refresh_rendered_content_from_s3(content_path, cache_key):
|
||||
""" Take a cache """
|
||||
result = get_content_from_s3(key=content_path)
|
||||
if result and result.get("content"):
|
||||
content = result.get("content")
|
||||
content_type = result.get("content_type")
|
||||
last_updated_at_raw = result.get("last_updated_at")
|
||||
def clear_rendered_content_cache_by_cache_key(cache_key):
|
||||
"""Deletes a RenderedContent object by its cache key from redis and
|
||||
database."""
|
||||
cache = caches["static_content"]
|
||||
cache.delete(cache_key)
|
||||
RenderedContent.objects.delete_by_cache_key(cache_key)
|
||||
|
||||
|
||||
@shared_task
|
||||
def clear_rendered_content_cache_by_content_type(content_type):
|
||||
"""Deletes all RenderedContent objects for a given content type from redis
|
||||
and database."""
|
||||
RenderedContent.objects.clear_cache_by_content_type(content_type)
|
||||
RenderedContent.objects.delete_by_content_type(content_type)
|
||||
|
||||
|
||||
@shared_task
|
||||
def refresh_content_from_s3(s3_key, cache_key):
|
||||
"""Calls S3 with the s3_key, then saves the result to the
|
||||
RenderedContent object with the given cache_key."""
|
||||
content_dict = get_content_from_s3(key=s3_key)
|
||||
content = content_dict.get("content")
|
||||
if content_dict and content:
|
||||
content_type = content_dict.get("content_type")
|
||||
if content_type == "text/asciidoc":
|
||||
content = self.convert_adoc_to_html(content, cache_key)
|
||||
last_updated_at = (
|
||||
parse(last_updated_at_raw) if last_updated_at_raw else None
|
||||
)
|
||||
content = process_adoc_to_html_content(content)
|
||||
last_updated_at_raw = content_dict.get("last_updated_at")
|
||||
last_updated_at = parse(last_updated_at_raw) if last_updated_at_raw else None
|
||||
# Clear the cache because we're going to update it.
|
||||
clear_rendered_content_cache_by_cache_key(cache_key)
|
||||
|
||||
# Get the output from the command
|
||||
converted_html = result.stdout
|
||||
# Update the rendered content.
|
||||
save_rendered_content(
|
||||
cache_key, content_type, content, last_updated_at=last_updated_at
|
||||
)
|
||||
# Cache the refreshed rendered content
|
||||
cache = caches["static_content"]
|
||||
cache.set(cache_key, {"content": content, "content_type": content_type})
|
||||
|
||||
# Delete the temporary file
|
||||
if delete_file:
|
||||
os.remove(file_path)
|
||||
|
||||
return converted_html
|
||||
@shared_task
|
||||
def save_rendered_content(cache_key, content_type, content_html, last_updated_at=None):
|
||||
"""Saves a RenderedContent object to database."""
|
||||
defaults = {
|
||||
"content_type": content_type,
|
||||
"content_html": content_html,
|
||||
}
|
||||
|
||||
if last_updated_at:
|
||||
defaults["last_updated_at"] = last_updated_at
|
||||
|
||||
obj, created = RenderedContent.objects.update_or_create(
|
||||
cache_key=cache_key[:255], defaults=defaults
|
||||
)
|
||||
logger.info(
|
||||
"content_saved_to_rendered_content",
|
||||
cache_key=cache_key,
|
||||
content_type=content_type,
|
||||
status_code=200,
|
||||
obj_id=obj.id,
|
||||
obj_created=created,
|
||||
)
|
||||
return obj
|
||||
|
||||
49
core/tests/test_asciidoc.py
Normal file
49
core/tests/test_asciidoc.py
Normal file
@@ -0,0 +1,49 @@
|
||||
import pytest
|
||||
import tempfile
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.core.cache import caches
|
||||
from django.test import override_settings
|
||||
|
||||
from core.asciidoc import convert_adoc_to_html, process_adoc_to_html_content
|
||||
|
||||
|
||||
@override_settings(
|
||||
CACHES={
|
||||
"static_content": {
|
||||
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
|
||||
"LOCATION": "third-unique-snowflake",
|
||||
"TIMEOUT": "60", # Cache timeout in seconds: 1 minute
|
||||
},
|
||||
}
|
||||
)
|
||||
def test_adoc_to_html():
|
||||
# Get the static content cache
|
||||
caches["static_content"]
|
||||
|
||||
# The content of the sample adoc file
|
||||
sample_adoc_content = "= Document Title\n\nThis is a sample document.\n"
|
||||
|
||||
# Write the content to a temporary file
|
||||
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
|
||||
temp_file.write(sample_adoc_content.encode())
|
||||
temp_file_path = temp_file.name
|
||||
|
||||
# Execute the task
|
||||
with patch("core.asciidoc.subprocess.run") as mock_run:
|
||||
mock_run.return_value.stdout = "html_content".encode()
|
||||
convert_adoc_to_html(temp_file_path, delete_file=True)
|
||||
|
||||
# Verify that the temporary file has been deleted
|
||||
with pytest.raises(FileNotFoundError):
|
||||
with open(temp_file_path, "r"):
|
||||
pass
|
||||
|
||||
|
||||
def test_process_adoc_to_html_content():
|
||||
"""Test the process_adoc_to_html_content function."""
|
||||
content = "sample"
|
||||
expected_html = '<div id="header">\n</div><div id="content">\n<div class="paragraph">\n<p>sample</p>\n</div>\n</div>' # noqa: E501
|
||||
|
||||
result = process_adoc_to_html_content(content)
|
||||
assert result == expected_html
|
||||
@@ -38,7 +38,7 @@ def test_adoc_to_html():
|
||||
temp_file_path = temp_file.name
|
||||
|
||||
# Execute the task
|
||||
with patch("core.tasks.subprocess.run") as mock_run:
|
||||
with patch("core.asciidoc.subprocess.run") as mock_run:
|
||||
mock_run.return_value.stdout = "html_content".encode()
|
||||
adoc_to_html(temp_file_path, delete_file=True)
|
||||
|
||||
|
||||
150
core/views.py
150
core/views.py
@@ -1,6 +1,5 @@
|
||||
import os.path
|
||||
import structlog
|
||||
import tempfile
|
||||
from dateutil.parser import parse
|
||||
|
||||
from django.conf import settings
|
||||
@@ -11,13 +10,14 @@ from django.views import View
|
||||
from django.views.generic import TemplateView
|
||||
|
||||
from .asciidoc import process_adoc_to_html_content
|
||||
from .boostrenderer import get_body_from_html, get_content_from_s3
|
||||
from .boostrenderer import get_content_from_s3
|
||||
from .markdown import process_md
|
||||
from .models import RenderedContent
|
||||
from .tasks import (
|
||||
adoc_to_html,
|
||||
clear_rendered_content_cache_by_cache_key,
|
||||
clear_rendered_content_cache_by_content_type,
|
||||
refresh_content_from_s3,
|
||||
save_rendered_content,
|
||||
)
|
||||
|
||||
logger = structlog.get_logger()
|
||||
@@ -164,12 +164,38 @@ class StaticContentTemplateView(TemplateView):
|
||||
return HttpResponseNotFound("Page not found")
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
def get_template_names(self):
|
||||
"""Returns the template name."""
|
||||
content_type = self.content_dict.get("content_type")
|
||||
if content_type == "text/asciidoc":
|
||||
return [self.template_name]
|
||||
return []
|
||||
def cache_result(self, static_content_cache, cache_key, result):
|
||||
static_content_cache.set(cache_key, result)
|
||||
|
||||
def get_content(self, content_path):
|
||||
"""Returns content from cache, database, or S3"""
|
||||
static_content_cache = caches["static_content"]
|
||||
cache_key = f"static_content_{content_path}"
|
||||
result = self.get_from_cache(static_content_cache, cache_key)
|
||||
|
||||
if result is None:
|
||||
result = self.get_from_database(cache_key)
|
||||
if result:
|
||||
# When we get a result from the database, we refresh its content
|
||||
refresh_content_from_s3.delay(content_path, cache_key)
|
||||
|
||||
if result is None:
|
||||
result = self.get_from_s3(content_path)
|
||||
if result:
|
||||
# Save to database
|
||||
self.save_to_database(cache_key, result)
|
||||
# Cache the result
|
||||
self.cache_result(static_content_cache, cache_key, result)
|
||||
|
||||
if result is None:
|
||||
logger.info(
|
||||
"get_content_from_s3_view_no_valid_object",
|
||||
key=content_path,
|
||||
status_code=404,
|
||||
)
|
||||
raise ContentNotFoundException("Content not found")
|
||||
|
||||
return result
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
"""Returns the content and content type for the template. In some cases,
|
||||
@@ -189,6 +215,36 @@ class StaticContentTemplateView(TemplateView):
|
||||
|
||||
return context
|
||||
|
||||
def get_from_cache(self, static_content_cache, cache_key):
|
||||
cached_result = static_content_cache.get(cache_key)
|
||||
return cached_result if cached_result else None
|
||||
|
||||
def get_from_database(self, cache_key):
|
||||
try:
|
||||
content_obj = RenderedContent.objects.get(cache_key=cache_key)
|
||||
return {
|
||||
"content": content_obj.content_html,
|
||||
"content_type": content_obj.content_type,
|
||||
}
|
||||
except RenderedContent.DoesNotExist:
|
||||
return None
|
||||
|
||||
def get_from_s3(self, content_path):
|
||||
result = get_content_from_s3(key=content_path)
|
||||
if result and result.get("content"):
|
||||
content = result.get("content")
|
||||
content_type = result.get("content_type")
|
||||
if content_type == "text/asciidoc":
|
||||
result["content"] = self.convert_adoc_to_html(content)
|
||||
return result
|
||||
|
||||
def get_template_names(self):
|
||||
"""Returns the template name."""
|
||||
content_type = self.content_dict.get("content_type")
|
||||
if content_type == "text/asciidoc":
|
||||
return [self.template_name]
|
||||
return []
|
||||
|
||||
def render_to_response(self, context, **response_kwargs):
|
||||
"""Return the HTML response with a template, or just the content directly."""
|
||||
if self.get_template_names():
|
||||
@@ -198,82 +254,22 @@ class StaticContentTemplateView(TemplateView):
|
||||
context["content"], content_type=context["content_type"]
|
||||
)
|
||||
|
||||
def get_content(self, content_path):
|
||||
"""Returns content from cache, database, or S3"""
|
||||
static_content_cache = caches["static_content"]
|
||||
cache_key = f"static_content_{content_path}"
|
||||
result = self.get_from_cache(static_content_cache, cache_key)
|
||||
|
||||
if result is None:
|
||||
result = self.get_from_database(cache_key)
|
||||
|
||||
if result is None:
|
||||
result = self.get_from_s3(content_path, cache_key)
|
||||
# Cache the result
|
||||
self.cache_result(static_content_cache, cache_key, result)
|
||||
|
||||
if result is None:
|
||||
logger.info(
|
||||
"get_content_from_s3_view_no_valid_object",
|
||||
key=content_path,
|
||||
status_code=404,
|
||||
)
|
||||
raise ContentNotFoundException("Content not found")
|
||||
|
||||
return result
|
||||
|
||||
def cache_result(self, static_content_cache, cache_key, result):
|
||||
static_content_cache.set(cache_key, result)
|
||||
|
||||
def get_from_cache(self, static_content_cache, cache_key):
|
||||
cached_result = static_content_cache.get(cache_key)
|
||||
return cached_result if cached_result else None
|
||||
|
||||
def get_from_database(self, cache_key):
|
||||
try:
|
||||
content_obj = RenderedContent.objects.get(cache_key=cache_key)
|
||||
# todo: fire refresh task here
|
||||
return {
|
||||
"content": content_obj.content_html,
|
||||
"content_type": content_obj.content_type,
|
||||
"last_updated_at": content_obj.last_updated_at,
|
||||
}
|
||||
except RenderedContent.DoesNotExist:
|
||||
return None
|
||||
|
||||
def get_from_s3(self, content_path, cache_key):
|
||||
result = get_content_from_s3(key=content_path)
|
||||
if result and result.get("content"):
|
||||
return self.update_or_create_content(result, cache_key)
|
||||
return
|
||||
|
||||
def update_or_create_content(self, result, cache_key):
|
||||
content = result.get("content")
|
||||
def save_to_database(self, cache_key, result):
|
||||
"""Saves the rendered asciidoc content to the database via celery."""
|
||||
content_type = result.get("content_type")
|
||||
last_updated_at_raw = result.get("last_updated_at")
|
||||
|
||||
if content_type == "text/asciidoc":
|
||||
content = self.convert_adoc_to_html(content)
|
||||
last_updated_at_raw = result.get("last_updated_at")
|
||||
last_updated_at = (
|
||||
parse(last_updated_at_raw) if last_updated_at_raw else None
|
||||
)
|
||||
|
||||
defaults = {"content_html": content, "content_type": content_type}
|
||||
if last_updated_at:
|
||||
defaults["last_updated_at"] = last_updated_at
|
||||
content_obj, created = RenderedContent.objects.update_or_create(
|
||||
cache_key=cache_key, defaults=defaults
|
||||
save_rendered_content.delay(
|
||||
cache_key,
|
||||
content_type,
|
||||
result["content"],
|
||||
last_updated_at=last_updated_at,
|
||||
)
|
||||
logger.info(
|
||||
"get_content_from_s3_view_saved_to_db",
|
||||
cache_key=cache_key,
|
||||
content_type=content_type,
|
||||
status_code=200,
|
||||
obj_id=content_obj.id,
|
||||
created=created,
|
||||
)
|
||||
result["content"] = content
|
||||
return result
|
||||
|
||||
def convert_adoc_to_html(self, content):
|
||||
"""Renders asciidoc content to HTML."""
|
||||
|
||||
Reference in New Issue
Block a user