From b49451e4a8431869074b3348d670a749fa7cb3d2 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 31 Mar 2023 10:29:25 -0700 Subject: [PATCH 1/5] :arrow_up: Add minio and boto3 --- requirements.in | 2 ++ requirements.txt | 14 +++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/requirements.in b/requirements.in index 1f469c13..aea15bf5 100755 --- a/requirements.in +++ b/requirements.in @@ -19,6 +19,7 @@ Pillow==9.4.0 django-storages wheel==0.38.1 cryptography==39.0.1 +boto3 # Logging django-tracer @@ -41,6 +42,7 @@ Faker model_bakery ipython responses +minio # Packaging pip-tools==6.12.3 diff --git a/requirements.txt b/requirements.txt index bc7867fc..797b8f5c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,9 @@ billiard==3.6.4.0 black==22.3 # via -r ./requirements.in boto3==1.17.68 - # via django-bakery + # via + # -r ./requirements.in + # django-bakery botocore==1.20.68 # via # boto3 @@ -39,7 +41,9 @@ bumpversion==0.6.0 celery==5.2.2 # via -r ./requirements.in certifi==2022.6.15 - # via requests + # via + # minio + # requests cffi==1.15.1 # via cryptography charset-normalizer==2.1.0 @@ -171,6 +175,8 @@ marshmallow==3.11.1 # via environs matplotlib-inline==0.1.3 # via ipython +minio==7.1.14 + # via -r ./requirements.in mistletoe==0.8.2 # via -r ./requirements.in model-bakery==1.5.0 @@ -277,7 +283,6 @@ s3transfer==0.4.2 # via boto3 six==1.16.0 # via - # asttokens # click-repl # django-bakery # django-rest-auth @@ -297,7 +302,6 @@ tomli==2.0.1 # via # black # build - # pyproject-hooks # pytest traitlets==5.2.1.post0 # via @@ -310,11 +314,11 @@ typing-extensions==4.5.0 urllib3==1.26.4 # via # botocore + # minio # requests # responses vine==5.0.0 # via - # amqp # celery # kombu wcwidth==0.2.5 From c69440fd83613bb4fbba802cb7c355f70e24fe5e Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Mon, 3 Apr 2023 12:30:04 -0700 Subject: [PATCH 2/5] AWS settings --- config/settings.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/settings.py b/config/settings.py index c830041b..9126d58d 100755 --- a/config/settings.py +++ b/config/settings.py @@ -354,6 +354,12 @@ if not LOCAL_DEVELOPMENT: DEFAULT_FILE_STORAGE = "core.storages.MediaStorage" MEDIA_URL = f"{AWS_S3_ENDPOINT_URL}/{MEDIA_BUCKET_NAME}/" +AWS_ACCESS_KEY_ID = env("DEV_AWS_ACCESS_KEY", default="changeme") +AWS_SECRET_ACCESS_KEY = env("DEV_AWS_SECRET_KEY", default="changeme") +DEV_BUCKET = env("DEV_BUCKET", default="changeme") +AWS_S3_ENDPOINT_URL = "s3.amazonaws.com" +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + # Markdown content BASE_CONTENT = env("BOOST_CONTENT_DIRECTORY", "/website") From eb213ffba1d65ee6a6618dac58390357973f653e Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Mon, 3 Apr 2023 12:30:33 -0700 Subject: [PATCH 3/5] WIP: Getting to know boto3 --- core/boostrenderer.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/boostrenderer.py b/core/boostrenderer.py index dd164c08..b02d530d 100644 --- a/core/boostrenderer.py +++ b/core/boostrenderer.py @@ -1,5 +1,8 @@ +import boto3 import re +from django.conf import settings + from mistletoe import HTMLRenderer from mistletoe.span_token import SpanToken from pygments import highlight @@ -8,6 +11,22 @@ from pygments.lexers import get_lexer_by_name as get_lexer, guess_lexer from pygments.formatters.html import HtmlFormatter +def get_content_from_s3(): + """Get content from S3 + + Sample key: + s3://stage.boost.org/archives/boost__76_0/tools/auto_index/doc/html/index/s07.html + + aws s3 cp s3://stage.boost.org/archives/boost__76_0/tools/auto_index/doc/html/index/s07.html Users/lacey/BoostArchive/s07.html + """ + s3 = boto3.client('s3', aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, region_name='us-east-1') + bucket_name = settings.DEV_BUCKET + key = 'archives/boost__81_0/README.md' + response = s3.get_object(Bucket=bucket_name, Key=key) + file_content = response['Body'].read().decode('utf-8') + return file_content + + class Youtube(SpanToken): """ Span token for Youtube shortcodes From 272061508e5eef664d46a91a0cc02b0ba2481a09 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Mon, 3 Apr 2023 13:18:21 -0700 Subject: [PATCH 4/5] :construction: Progress on s3 retrieval --- config/settings.py | 2 +- core/boostrenderer.py | 33 +++++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/config/settings.py b/config/settings.py index 9126d58d..0b8de209 100755 --- a/config/settings.py +++ b/config/settings.py @@ -358,7 +358,7 @@ AWS_ACCESS_KEY_ID = env("DEV_AWS_ACCESS_KEY", default="changeme") AWS_SECRET_ACCESS_KEY = env("DEV_AWS_SECRET_KEY", default="changeme") DEV_BUCKET = env("DEV_BUCKET", default="changeme") AWS_S3_ENDPOINT_URL = "s3.amazonaws.com" -DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" # Markdown content BASE_CONTENT = env("BOOST_CONTENT_DIRECTORY", "/website") diff --git a/core/boostrenderer.py b/core/boostrenderer.py index b02d530d..6838fd53 100644 --- a/core/boostrenderer.py +++ b/core/boostrenderer.py @@ -11,19 +11,32 @@ from pygments.lexers import get_lexer_by_name as get_lexer, guess_lexer from pygments.formatters.html import HtmlFormatter -def get_content_from_s3(): +def get_content_from_s3(key=None, bucket_name=None): """Get content from S3 - - Sample key: - s3://stage.boost.org/archives/boost__76_0/tools/auto_index/doc/html/index/s07.html - aws s3 cp s3://stage.boost.org/archives/boost__76_0/tools/auto_index/doc/html/index/s07.html Users/lacey/BoostArchive/s07.html + Sample key: + 'archives/boost_1_81_0/README.md' + + Returns the decoded file contents if able + + FIXME: This is a temporary solution to get the content from S3 + and does not handle errors or anything unexpected with grace. """ - s3 = boto3.client('s3', aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, region_name='us-east-1') - bucket_name = settings.DEV_BUCKET - key = 'archives/boost__81_0/README.md' - response = s3.get_object(Bucket=bucket_name, Key=key) - file_content = response['Body'].read().decode('utf-8') + + if not key: + raise + + if not bucket_name: + bucket_name = settings.DEV_BUCKET + + client = boto3.client( + "s3", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + region_name="us-east-1", + ) + response = client.get_object(Bucket=bucket_name, Key=key) + file_content = response["Body"].read().decode("utf-8") return file_content From c359f325cd0c265a69a316e653d66471ca703ccc Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Tue, 4 Apr 2023 09:23:04 -0700 Subject: [PATCH 5/5] :snowflake: Change setting names for AWS --- config/settings.py | 6 +++--- core/boostrenderer.py | 2 +- env.template | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/config/settings.py b/config/settings.py index 0b8de209..c46debcd 100755 --- a/config/settings.py +++ b/config/settings.py @@ -354,9 +354,9 @@ if not LOCAL_DEVELOPMENT: DEFAULT_FILE_STORAGE = "core.storages.MediaStorage" MEDIA_URL = f"{AWS_S3_ENDPOINT_URL}/{MEDIA_BUCKET_NAME}/" -AWS_ACCESS_KEY_ID = env("DEV_AWS_ACCESS_KEY", default="changeme") -AWS_SECRET_ACCESS_KEY = env("DEV_AWS_SECRET_KEY", default="changeme") -DEV_BUCKET = env("DEV_BUCKET", default="changeme") +AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID", default="changeme") +AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY", default="changeme") +BUCKET_NAME = env("BUCKET_NAME", default="changeme") AWS_S3_ENDPOINT_URL = "s3.amazonaws.com" DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" diff --git a/core/boostrenderer.py b/core/boostrenderer.py index 6838fd53..49206915 100644 --- a/core/boostrenderer.py +++ b/core/boostrenderer.py @@ -27,7 +27,7 @@ def get_content_from_s3(key=None, bucket_name=None): raise if not bucket_name: - bucket_name = settings.DEV_BUCKET + bucket_name = settings.BUCKET_NAME client = boto3.client( "s3", diff --git a/env.template b/env.template index 985eea19..cfa5aae0 100644 --- a/env.template +++ b/env.template @@ -15,3 +15,8 @@ DJANGO_DEBUG=1 SECRET_KEY="top-secret" GITHUB_TOKEN="top-secret" + +AWS_ACCESS_KEY_ID="changeme" +AWS_SECRET_ACCESS_KEY="changeme" +BUCKET_NAME="stage.boost.org" +