From 7dea27d1d1edc2fd0a5c3a1e83687139934d5934 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 10:46:29 -0800 Subject: [PATCH 01/16] :bank: Add valid_email and claimed fields to User model Default to `True` because the only conditions in which these would be False is on import of author/maintainer data for the Libraries from GitHub. After the initial launch, these fields won't be used with nearly the frequency. --- users/migrations/0008_auto_20230303_1843.py | 27 +++++++++++++++++++++ users/models.py | 8 +++++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 users/migrations/0008_auto_20230303_1843.py diff --git a/users/migrations/0008_auto_20230303_1843.py b/users/migrations/0008_auto_20230303_1843.py new file mode 100644 index 00000000..0f116a8f --- /dev/null +++ b/users/migrations/0008_auto_20230303_1843.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.2 on 2023-03-03 18:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0007_user_image"), + ] + + operations = [ + migrations.AddField( + model_name="user", + name="claimed", + field=models.BooleanField( + default=True, verbose_name="whether this account has been claimed" + ), + ), + migrations.AddField( + model_name="user", + name="valid_email", + field=models.BooleanField( + default=True, verbose_name="whether the user's email address is valid" + ), + ), + ] diff --git a/users/models.py b/users/models.py index 05f1b759..2aec56ab 100644 --- a/users/models.py +++ b/users/models.py @@ -119,7 +119,7 @@ class BaseUser(AbstractBaseUser, PermissionsMixin): return full_name.strip() def get_short_name(self): - "Returns the short name for the user." + """Returns the short name for the user.""" return self.first_name def email_user(self, subject, message, from_email=None, **kwargs): @@ -150,6 +150,12 @@ class User(BaseUser): badges = models.ManyToManyField(Badge) github_username = models.CharField(_("github username"), max_length=100, blank=True) image = models.FileField(upload_to="profile-images", null=True, blank=True) + valid_email = models.BooleanField( + _("whether the user's email address is valid"), default=True + ) + claimed = models.BooleanField( + _("whether this account has been claimed"), default=True + ) def save_image_from_github(self, avatar_url): response = requests.get(avatar_url) From 6e07a8946248b059b7d044a5219eaeacd4d4311c Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 13:11:25 -0800 Subject: [PATCH 02/16] :pencil: Change help text on new fields --- users/migrations/0009_auto_20230303_2102.py | 25 +++++++++++++++++++++ users/models.py | 8 ++----- 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 users/migrations/0009_auto_20230303_2102.py diff --git a/users/migrations/0009_auto_20230303_2102.py b/users/migrations/0009_auto_20230303_2102.py new file mode 100644 index 00000000..fbc72a3f --- /dev/null +++ b/users/migrations/0009_auto_20230303_2102.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.2 on 2023-03-03 21:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0008_auto_20230303_1843"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="claimed", + field=models.BooleanField( + default=True, verbose_name="Account has been claimed" + ), + ), + migrations.AlterField( + model_name="user", + name="valid_email", + field=models.BooleanField(default=True, verbose_name="Valid email address"), + ), + ] diff --git a/users/models.py b/users/models.py index 2aec56ab..1ca6e60a 100644 --- a/users/models.py +++ b/users/models.py @@ -150,12 +150,8 @@ class User(BaseUser): badges = models.ManyToManyField(Badge) github_username = models.CharField(_("github username"), max_length=100, blank=True) image = models.FileField(upload_to="profile-images", null=True, blank=True) - valid_email = models.BooleanField( - _("whether the user's email address is valid"), default=True - ) - claimed = models.BooleanField( - _("whether this account has been claimed"), default=True - ) + valid_email = models.BooleanField(_("Valid email address"), default=True) + claimed = models.BooleanField(_("Account has been claimed"), default=True) def save_image_from_github(self, avatar_url): response = requests.get(avatar_url) From bb4bd377134616c2a3baf0af4b35b1bdb32112c1 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 13:11:46 -0800 Subject: [PATCH 03/16] :bank: Move maintainers M2M to LibraryVersion --- .../migrations/0005_auto_20230303_2100.py | 26 +++++++++++++++++++ libraries/models.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 libraries/migrations/0005_auto_20230303_2100.py diff --git a/libraries/migrations/0005_auto_20230303_2100.py b/libraries/migrations/0005_auto_20230303_2100.py new file mode 100644 index 00000000..c2af8a02 --- /dev/null +++ b/libraries/migrations/0005_auto_20230303_2100.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.2 on 2023-03-03 21:00 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("libraries", "0004_auto_20230130_1830"), + ] + + operations = [ + migrations.RemoveField( + model_name="library", + name="maintainers", + ), + migrations.AddField( + model_name="libraryversion", + name="maintainers", + field=models.ManyToManyField( + related_name="maintainers", to=settings.AUTH_USER_MODEL + ), + ), + ] diff --git a/libraries/models.py b/libraries/models.py index 442bc6bc..4288498a 100644 --- a/libraries/models.py +++ b/libraries/models.py @@ -54,7 +54,6 @@ class Library(models.Model): categories = models.ManyToManyField(Category, related_name="libraries") authors = models.ManyToManyField("users.User", related_name="authors") - maintainers = models.ManyToManyField("users.User", related_name="maintainers") closed_prs_per_month = models.IntegerField(blank=True, null=True) open_issues = models.IntegerField(blank=True, null=True) @@ -105,6 +104,7 @@ class LibraryVersion(models.Model): on_delete=models.SET_NULL, null=True, ) + maintainers = models.ManyToManyField("users.User", related_name="maintainers") def __str__(self): return f"{self.library.name} ({self.version.name})" From 2497be4eaa26e5d2364efb4c796f37da37f0e4d3 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 13:12:03 -0800 Subject: [PATCH 04/16] :sparkles: Add ordering to admin --- libraries/admin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/admin.py b/libraries/admin.py index 547411b8..30524bcb 100644 --- a/libraries/admin.py +++ b/libraries/admin.py @@ -6,6 +6,7 @@ from .models import Category, Issue, Library, LibraryVersion, PullRequest @admin.register(Category) class CategoryAdmin(admin.ModelAdmin): list_display = ["name"] + ordering = ["name"] search_fields = ["name"] @@ -14,6 +15,7 @@ class LibraryAdmin(admin.ModelAdmin): list_display = ["name", "active", "open_issues"] search_fields = ["name", "description"] list_filter = ["active_development", "categories"] + ordering = ["name"] readonly_fields = [ "last_github_update", @@ -30,6 +32,7 @@ class LibraryAdmin(admin.ModelAdmin): class LibraryVersionAdmin(admin.ModelAdmin): list_display = ["library", "version"] list_filter = ["library", "version"] + ordering = ["library__name", "-version__name"] search_fields = ["library__name", "version__name"] From f8c6d2510d8854c342691ceedd3f55538681437a Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 13:12:41 -0800 Subject: [PATCH 05/16] :sparkles: Add new fields --- users/admin.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/users/admin.py b/users/admin.py index 8148c8e7..e8df950d 100644 --- a/users/admin.py +++ b/users/admin.py @@ -10,7 +10,15 @@ class EmailUserAdmin(UserAdmin): (None, {"fields": ("email", "password")}), ( _("Personal info"), - {"fields": ("first_name", "last_name", "github_username")}, + { + "fields": ( + "first_name", + "last_name", + "github_username", + "valid_email", + "claimed", + ) + }, ), ( _("Permissions"), @@ -32,8 +40,15 @@ class EmailUserAdmin(UserAdmin): (None, {"classes": ("wide",), "fields": ("email", "password1", "password2")}), ) ordering = ("email",) - list_display = ("email", "first_name", "last_name", "is_staff") - search_fields = ("email" "first_name", "last_name") + list_display = ( + "email", + "first_name", + "last_name", + "is_staff", + "valid_email", + "claimed", + ) + search_fields = ("email", "first_name", "last_name") admin.site.register(User, EmailUserAdmin) From f97e2c44b53e8b34f0d15954f2fd85e9f4aeb1b9 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 13:16:30 -0800 Subject: [PATCH 06/16] :construction: Command to upload lib authors and maintainers WIP. Uses a single, randomly-retrieved library to test. - Gets the library GH libraries.json - Extracts author and maintainer data - For each person: - Extracts the email if available, or autogenerates a placeholder and marks the email as invalid - Separates the name fields into First and Last - For authors, adds the author's User record to the Library's `authors` - For maintainers, adds the maintainer's User record to the LibraryVersion's `maintainers` (using the most recent Version) --- .../commands/create_authors_maintainers.py | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 libraries/management/commands/create_authors_maintainers.py diff --git a/libraries/management/commands/create_authors_maintainers.py b/libraries/management/commands/create_authors_maintainers.py new file mode 100644 index 00000000..2f63db61 --- /dev/null +++ b/libraries/management/commands/create_authors_maintainers.py @@ -0,0 +1,197 @@ +import djclick as click +import random +import re +import structlog + +from datetime import timedelta +from django.contrib.auth import get_user_model +from django.core.exceptions import ValidationError +from django.core.validators import validate_email +from django.utils import timezone +from django.utils.text import slugify +from faker import Faker + +from libraries.github import LibraryUpdater +from libraries.models import Library, LibraryVersion +from versions.models import Version + +fake = Faker() + +logger = structlog.get_logger() + +User = get_user_model() + + +EMAIL = "Tester Testerston " +NAME = "Tester de Testerson" + + +@click.command() +def command(): + """ + Idempotent. + + Get all the libraries. + + For each library: + - Retrieve and extract info about their authors and maintainers + - Use that info to create or update User records + - Associate the User records for those authors and maintainers to the + Library + + - [x] Retrieve their GH record + - [x] Get the file with the data in it and parse it + - [x] Extract author and maintainer data + - [x] Method to extract email + - [x] Method to extract first and last + - [x] If no email, method to create fake email + - [x] Add a User model field to denote a fake email + - [x] Add a User model field to denote unclaimed user + - [x] Method to get or create new user with email + - [x] If created, mark as Unclaimed + - [x] If fake email, mark fake email + - [x] Add the authors as Authors to the Library + + # Dealing with maintainers + - [x] Remove 'maintainers' from Library and add to LibraryVersion + - [x] Retrieve most recent LibraryVersion + - [x] Add maintainers to the most recent LibraryVersion + + NEW ISSUES + - [ ] Process to claim your user with your email + - [ ] Add process to update authors to the library syncing + - [ ] Add process to update maintainers to the LibraryVersion syncing + """ + + library = Library.objects.order_by("?").first() + version = Version.objects.most_recent() + click.secho(f"Getting Library data for '{library.name}'...", fg="green") + + updater = LibraryUpdater() + result = updater.get_library_metadata(library.name) + authors = result.get("authors") + maintainers = result.get("maintainers") + + click.secho(f"Getting authors...", fg="green") + for a in authors: + person_data = get_person_data(a) + user, created = User.objects.get_or_create( + email=person_data["email"], + defaults={ + "first_name": person_data["first_name"][:30], + "last_name": person_data["last_name"][:30], + }, + ) + click.secho(f"User {user.email} saved. Created? {created}", fg="green") + + if created: + user.claimed = False + user.is_active = False + user.save() + + library.authors.add(user) + click.secho( + f"User {user.email} added as a maintainer of {library.name}", fg="green" + ) + + if maintainers: + try: + library_version = LibraryVersion.objects.get(library=library, version=version) + except LibraryVersion.DoesNotExist: + logger.info("No library version", version=version.pk, library=library.pk) + return + + click.secho(f"Getting maintainers...", fg="green") + for m in maintainers: + person_data = get_person_data(m) + + user, created = User.objects.get_or_create( + email=person_data["email"], + defaults={ + "first_name": person_data["first_name"][:30], + "last_name": person_data["last_name"][:30], + }, + ) + click.secho(f"--User {user.email} saved. Created? {created}", fg="green") + + if created: + user.claimed = False + user.is_active = False + user.valid_email = person_data["valid_email"] + user.save() + + library_version.maintainers.add(user) + click.secho( + f"--User {user.email} added as a maintainer of {library.name}", fg="green" + ) + + click.secho("All done!", fg="green") + + +def get_person_data(person: str) -> dict: + """Takes an author/maintainer string and returns a dict with their data""" + person_data = {} + person_data["meta"] = person + person_data["valid_email"] = False + + email = get_email(person) + if email: + person_data["email"] = email + person_data["valid_email"] = True + else: + person_data["email"] = generate_email(person) + + names = get_names(person) + person_data["first_name"] = names[0] + person_data["last_name"] = names[1] + + return person_data + + +def get_email(val: str) -> str: + """ + Finds an email address in a string, reformats it, and returns it. + + Assumes the email address is in this format: + + """ + result = re.search("<.+>", val) + if result: + raw_email = result.group() + email = ( + raw_email.replace("-at-", "@") + .replace("<", "") + .replace(">", "") + .replace(" ", "") + ) + try: + validate_email(email) + except ValidationError as e: + # TODO: Output this to a list of some sort + logger.info("Could not extract valid email", value=val) + return + return email + + +def get_names(val: str) -> list: + """ + Returns a ,list of first, last names for the val argument. + + NOTE: This is an overly simplistic solution to importing names. + Names that don't conform neatly to "First Last" formats will need + to be cleaned up manually. + """ + return val.rsplit(" ", 1) + + +def generate_email(val: str) -> str: + """Slugifies a string""" + slug = slugify(val) + local_email = slug.replace("-", "_")[:64] + email = f"{local_email}@example.com" + + if User.objects.filter(email=email).exists(): + # TODO: Deal with duplicates that are probably real. + pass + + return email From ffddd0afa28768f6f7401b0051ce6107f5ca666a Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 13:18:02 -0800 Subject: [PATCH 07/16] :bug: Fix conditional Can't check for an image URL if there is no image! --- templates/libraries/_library_list_item.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/libraries/_library_list_item.html b/templates/libraries/_library_list_item.html index d97803a2..c59c73eb 100644 --- a/templates/libraries/_library_list_item.html +++ b/templates/libraries/_library_list_item.html @@ -24,7 +24,7 @@
{% for author in library.authors.all %}
- {% if author.image.url %} + {% if author.image %} user {% else %} From f9247e7d244cb7a313d4f93fa41186886c549f46 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 13:47:18 -0800 Subject: [PATCH 08/16] :wrench: Start moving things to testable util functions - Move extract_email to util - Correct get_names method to strip the formatted email from the string before parsing --- .../commands/create_authors_maintainers.py | 54 +++++++++---------- libraries/tests/test_utils.py | 14 ++++- libraries/utils.py | 29 ++++++++++ 3 files changed, 66 insertions(+), 31 deletions(-) diff --git a/libraries/management/commands/create_authors_maintainers.py b/libraries/management/commands/create_authors_maintainers.py index 2f63db61..655630d3 100644 --- a/libraries/management/commands/create_authors_maintainers.py +++ b/libraries/management/commands/create_authors_maintainers.py @@ -11,6 +11,7 @@ from django.utils import timezone from django.utils.text import slugify from faker import Faker +from libraries import utils from libraries.github import LibraryUpdater from libraries.models import Library, LibraryVersion from versions.models import Version @@ -69,6 +70,12 @@ def command(): updater = LibraryUpdater() result = updater.get_library_metadata(library.name) + + if type(result) is list: + breakpoint() + # See line 211 in github.py + raise + authors = result.get("authors") maintainers = result.get("maintainers") @@ -96,7 +103,9 @@ def command(): if maintainers: try: - library_version = LibraryVersion.objects.get(library=library, version=version) + library_version = LibraryVersion.objects.get( + library=library, version=version + ) except LibraryVersion.DoesNotExist: logger.info("No library version", version=version.pk, library=library.pk) return @@ -122,7 +131,8 @@ def command(): library_version.maintainers.add(user) click.secho( - f"--User {user.email} added as a maintainer of {library.name}", fg="green" + f"--User {user.email} added as a maintainer of {library.name}", + fg="green", ) click.secho("All done!", fg="green") @@ -134,7 +144,7 @@ def get_person_data(person: str) -> dict: person_data["meta"] = person person_data["valid_email"] = False - email = get_email(person) + email = utils.extract_email(person) if email: person_data["email"] = email person_data["valid_email"] = True @@ -148,32 +158,9 @@ def get_person_data(person: str) -> dict: return person_data -def get_email(val: str) -> str: - """ - Finds an email address in a string, reformats it, and returns it. - - Assumes the email address is in this format: - - """ - result = re.search("<.+>", val) - if result: - raw_email = result.group() - email = ( - raw_email.replace("-at-", "@") - .replace("<", "") - .replace(">", "") - .replace(" ", "") - ) - try: - validate_email(email) - except ValidationError as e: - # TODO: Output this to a list of some sort - logger.info("Could not extract valid email", value=val) - return - return email - - -def get_names(val: str) -> list: +def get_names( + val: str, +) -> list: """ Returns a ,list of first, last names for the val argument. @@ -181,7 +168,14 @@ def get_names(val: str) -> list: Names that don't conform neatly to "First Last" formats will need to be cleaned up manually. """ - return val.rsplit(" ", 1) + # Strip the email, if present + email = re.search("<.+>", val) + if email: + # breakpoint() + val = val.replace(email.group(), "") + + # breakpoint() + return val.strip().rsplit(" ", 1) def generate_email(val: str) -> str: diff --git a/libraries/tests/test_utils.py b/libraries/tests/test_utils.py index 3f8b91de..03e1ad01 100644 --- a/libraries/tests/test_utils.py +++ b/libraries/tests/test_utils.py @@ -1,6 +1,18 @@ from datetime import datetime -from libraries.utils import parse_date +from libraries.utils import extract_email, parse_date + + +def test_extract_email(): + expected = "t_testerson@example.com" + result = extract_email("Tester Testerston ") + assert expected == result + + +def test_extract_email_no_email(): + expected = None + result = extract_email("Tester Testeron") + assert expected == result def test_parse_date_iso(): diff --git a/libraries/utils.py b/libraries/utils.py index 6ba1b1ef..78421106 100644 --- a/libraries/utils.py +++ b/libraries/utils.py @@ -1,7 +1,11 @@ +import re import structlog from dateutil.parser import ParserError, parse +from django.core.exceptions import ValidationError +from django.core.validators import validate_email + logger = structlog.get_logger() @@ -12,3 +16,28 @@ def parse_date(date_str): except ParserError: logger.info("parse_date_invalid_date", date_str=date_str) return None + + +def extract_email(val: str) -> str: + """ + Finds an email address in a string, reformats it, and returns it. + + Assumes the email address is in this format: + + """ + result = re.search("<.+>", val) + if result: + raw_email = result.group() + email = ( + raw_email.replace("-at-", "@") + .replace("<", "") + .replace(">", "") + .replace(" ", "") + ) + try: + validate_email(email) + except ValidationError as e: + # TODO: Output this to a list of some sort + logger.info("Could not extract valid email", value=val) + return + return email From 8baaaedc8f28aef1ca16d83e234fac6eb8340574 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 13:48:44 -0800 Subject: [PATCH 09/16] :wrench: Reorder functions --- libraries/utils.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/libraries/utils.py b/libraries/utils.py index 78421106..eddb9512 100644 --- a/libraries/utils.py +++ b/libraries/utils.py @@ -3,19 +3,14 @@ import structlog from dateutil.parser import ParserError, parse +from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.core.validators import validate_email +from django.utils.text import slugify logger = structlog.get_logger() - -def parse_date(date_str): - """Parses a date string to a datetime. Does not return an error.""" - try: - return parse(date_str) - except ParserError: - logger.info("parse_date_invalid_date", date_str=date_str) - return None +User = get_user_model() def extract_email(val: str) -> str: @@ -41,3 +36,12 @@ def extract_email(val: str) -> str: logger.info("Could not extract valid email", value=val) return return email + + +def parse_date(date_str): + """Parses a date string to a datetime. Does not return an error.""" + try: + return parse(date_str) + except ParserError: + logger.info("parse_date_invalid_date", date_str=date_str) + return None From 54bb53407b6204ff0ea33ecfdf8930b27129bcf2 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 13:53:49 -0800 Subject: [PATCH 10/16] :wrench: Move generate_email to utils --- .../commands/create_authors_maintainers.py | 27 ++++--------------- libraries/tests/test_utils.py | 8 +++++- libraries/utils.py | 7 +++++ 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/libraries/management/commands/create_authors_maintainers.py b/libraries/management/commands/create_authors_maintainers.py index 655630d3..7aca1720 100644 --- a/libraries/management/commands/create_authors_maintainers.py +++ b/libraries/management/commands/create_authors_maintainers.py @@ -5,8 +5,6 @@ import structlog from datetime import timedelta from django.contrib.auth import get_user_model -from django.core.exceptions import ValidationError -from django.core.validators import validate_email from django.utils import timezone from django.utils.text import slugify from faker import Faker @@ -144,16 +142,16 @@ def get_person_data(person: str) -> dict: person_data["meta"] = person person_data["valid_email"] = False + names = get_names(person) + person_data["first_name"] = names[0] + person_data["last_name"] = names[1] + email = utils.extract_email(person) if email: person_data["email"] = email person_data["valid_email"] = True else: - person_data["email"] = generate_email(person) - - names = get_names(person) - person_data["first_name"] = names[0] - person_data["last_name"] = names[1] + person_data["email"] = utils.generate_email(f"{person_data['first_name']} {person_data['last_name']}") return person_data @@ -171,21 +169,6 @@ def get_names( # Strip the email, if present email = re.search("<.+>", val) if email: - # breakpoint() val = val.replace(email.group(), "") - # breakpoint() return val.strip().rsplit(" ", 1) - - -def generate_email(val: str) -> str: - """Slugifies a string""" - slug = slugify(val) - local_email = slug.replace("-", "_")[:64] - email = f"{local_email}@example.com" - - if User.objects.filter(email=email).exists(): - # TODO: Deal with duplicates that are probably real. - pass - - return email diff --git a/libraries/tests/test_utils.py b/libraries/tests/test_utils.py index 03e1ad01..fd9b1350 100644 --- a/libraries/tests/test_utils.py +++ b/libraries/tests/test_utils.py @@ -1,6 +1,6 @@ from datetime import datetime -from libraries.utils import extract_email, parse_date +from libraries.utils import extract_email, generate_email, parse_date def test_extract_email(): @@ -15,6 +15,12 @@ def test_extract_email_no_email(): assert expected == result +def test_generate_email(): + expected = "tester_testerson@example.com" + result = generate_email("Tester Testerson") + assert expected == result + + def test_parse_date_iso(): expected = datetime.now() result = parse_date(expected.isoformat()) diff --git a/libraries/utils.py b/libraries/utils.py index eddb9512..fe2b2bcd 100644 --- a/libraries/utils.py +++ b/libraries/utils.py @@ -38,6 +38,13 @@ def extract_email(val: str) -> str: return email +def generate_email(val: str) -> str: + """ Takes a string and generates a placeholder email based on it """ + slug = slugify(val) + local_email = slug.replace("-", "_")[:64] + return f"{local_email}@example.com" + + def parse_date(date_str): """Parses a date string to a datetime. Does not return an error.""" try: From da1c28c64bf2844889c4916c85ed0687afd51b5d Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 3 Mar 2023 14:01:44 -0800 Subject: [PATCH 11/16] :wrench: Move name extraction to utils --- .../commands/create_authors_maintainers.py | 30 ++++--------------- libraries/tests/test_utils.py | 23 +++++++++++++- libraries/utils.py | 23 ++++++++++++-- 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/libraries/management/commands/create_authors_maintainers.py b/libraries/management/commands/create_authors_maintainers.py index 7aca1720..a73003f6 100644 --- a/libraries/management/commands/create_authors_maintainers.py +++ b/libraries/management/commands/create_authors_maintainers.py @@ -21,10 +21,6 @@ logger = structlog.get_logger() User = get_user_model() -EMAIL = "Tester Testerston " -NAME = "Tester de Testerson" - - @click.command() def command(): """ @@ -71,7 +67,7 @@ def command(): if type(result) is list: breakpoint() - # See line 211 in github.py + # TODO: See line 211 in github.py raise authors = result.get("authors") @@ -142,7 +138,7 @@ def get_person_data(person: str) -> dict: person_data["meta"] = person person_data["valid_email"] = False - names = get_names(person) + names = utils.get_names(person) person_data["first_name"] = names[0] person_data["last_name"] = names[1] @@ -151,24 +147,8 @@ def get_person_data(person: str) -> dict: person_data["email"] = email person_data["valid_email"] = True else: - person_data["email"] = utils.generate_email(f"{person_data['first_name']} {person_data['last_name']}") + person_data["email"] = utils.generate_email( + f"{person_data['first_name']} {person_data['last_name']}" + ) return person_data - - -def get_names( - val: str, -) -> list: - """ - Returns a ,list of first, last names for the val argument. - - NOTE: This is an overly simplistic solution to importing names. - Names that don't conform neatly to "First Last" formats will need - to be cleaned up manually. - """ - # Strip the email, if present - email = re.search("<.+>", val) - if email: - val = val.replace(email.group(), "") - - return val.strip().rsplit(" ", 1) diff --git a/libraries/tests/test_utils.py b/libraries/tests/test_utils.py index fd9b1350..7940a181 100644 --- a/libraries/tests/test_utils.py +++ b/libraries/tests/test_utils.py @@ -1,21 +1,42 @@ from datetime import datetime -from libraries.utils import extract_email, generate_email, parse_date +from libraries.utils import extract_email, extract_names, generate_email, parse_date def test_extract_email(): + """When formatted email is present in the string, expects a properly-formatted email address""" expected = "t_testerson@example.com" result = extract_email("Tester Testerston ") assert expected == result def test_extract_email_no_email(): + """When no email is present in the string, expects None""" expected = None result = extract_email("Tester Testeron") assert expected == result +def test_extract_names(): + """Expects a list of names""" + expected = ["Tester", "Testerson"] + result = extract_names("Tester Testerson") + assert expected == result + + expected = ["Tester de la", "Testerson"] + result = extract_names("Tester de la Testerson") + assert expected == result + + +def test_extract_names_with_email(): + """When the string contains email, expects that to be stripped out""" + expected = ["Tester", "Testerson"] + result = extract_names("Tester Testerson ") + assert expected == result + + def test_generate_email(): + """Given a string, expects a properly-formatted email address""" expected = "tester_testerson@example.com" result = generate_email("Tester Testerson") assert expected == result diff --git a/libraries/utils.py b/libraries/utils.py index fe2b2bcd..1c969234 100644 --- a/libraries/utils.py +++ b/libraries/utils.py @@ -32,14 +32,33 @@ def extract_email(val: str) -> str: try: validate_email(email) except ValidationError as e: - # TODO: Output this to a list of some sort logger.info("Could not extract valid email", value=val) return return email +def extract_names(name_str: str) -> list: + """ + Returns a ,list of first, last names for the val argument. + + NOTE: This is an overly simplistic solution to importing names. + Names that don't conform neatly to "First Last" formats will need + to be cleaned up manually. + + Expects something similar to these formats: + - "Tester Testerston " + - "Tester de Testerson" + """ + # Strip the email, if present + email = re.search("<.+>", name_str) + if email: + name_str = name_str.replace(email.group(), "") + + return name_str.strip().rsplit(" ", 1) + + def generate_email(val: str) -> str: - """ Takes a string and generates a placeholder email based on it """ + """Takes a string and generates a placeholder email based on it""" slug = slugify(val) local_email = slug.replace("-", "_")[:64] return f"{local_email}@example.com" From 3036d308b266b8b9310aa12f9ccb466f5b0aefe0 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Tue, 7 Mar 2023 13:38:28 -0800 Subject: [PATCH 12/16] :bug: Fix checking for URL if there is no image --- templates/libraries/detail.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/libraries/detail.html b/templates/libraries/detail.html index f102b74a..2935e768 100644 --- a/templates/libraries/detail.html +++ b/templates/libraries/detail.html @@ -83,7 +83,7 @@
{% for author in object.authors.all %}
- {% if author.image.url %} + {% if author.image %} user {% else %} @@ -96,9 +96,9 @@

Maintainers

- {% for maintainer in object.maintainers.all %} + {% for maintainer in maintainers.all %}
- {% if maintainer.image.url %} + {% if maintainer.image %} user {% else %} From e9a90fa54888cea69432549e766a228bc65dbe3f Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 10 Mar 2023 10:02:32 -0800 Subject: [PATCH 13/16] :fire: Remove old code, merged in another PR --- libraries/tests/test_utils.py | 41 +------------------------ libraries/utils.py | 58 ----------------------------------- 2 files changed, 1 insertion(+), 98 deletions(-) diff --git a/libraries/tests/test_utils.py b/libraries/tests/test_utils.py index 7940a181..3f8b91de 100644 --- a/libraries/tests/test_utils.py +++ b/libraries/tests/test_utils.py @@ -1,45 +1,6 @@ from datetime import datetime -from libraries.utils import extract_email, extract_names, generate_email, parse_date - - -def test_extract_email(): - """When formatted email is present in the string, expects a properly-formatted email address""" - expected = "t_testerson@example.com" - result = extract_email("Tester Testerston ") - assert expected == result - - -def test_extract_email_no_email(): - """When no email is present in the string, expects None""" - expected = None - result = extract_email("Tester Testeron") - assert expected == result - - -def test_extract_names(): - """Expects a list of names""" - expected = ["Tester", "Testerson"] - result = extract_names("Tester Testerson") - assert expected == result - - expected = ["Tester de la", "Testerson"] - result = extract_names("Tester de la Testerson") - assert expected == result - - -def test_extract_names_with_email(): - """When the string contains email, expects that to be stripped out""" - expected = ["Tester", "Testerson"] - result = extract_names("Tester Testerson ") - assert expected == result - - -def test_generate_email(): - """Given a string, expects a properly-formatted email address""" - expected = "tester_testerson@example.com" - result = generate_email("Tester Testerson") - assert expected == result +from libraries.utils import parse_date def test_parse_date_iso(): diff --git a/libraries/utils.py b/libraries/utils.py index 1c969234..07e2a2c0 100644 --- a/libraries/utils.py +++ b/libraries/utils.py @@ -1,68 +1,10 @@ -import re import structlog from dateutil.parser import ParserError, parse -from django.contrib.auth import get_user_model -from django.core.exceptions import ValidationError -from django.core.validators import validate_email -from django.utils.text import slugify logger = structlog.get_logger() -User = get_user_model() - - -def extract_email(val: str) -> str: - """ - Finds an email address in a string, reformats it, and returns it. - - Assumes the email address is in this format: - - """ - result = re.search("<.+>", val) - if result: - raw_email = result.group() - email = ( - raw_email.replace("-at-", "@") - .replace("<", "") - .replace(">", "") - .replace(" ", "") - ) - try: - validate_email(email) - except ValidationError as e: - logger.info("Could not extract valid email", value=val) - return - return email - - -def extract_names(name_str: str) -> list: - """ - Returns a ,list of first, last names for the val argument. - - NOTE: This is an overly simplistic solution to importing names. - Names that don't conform neatly to "First Last" formats will need - to be cleaned up manually. - - Expects something similar to these formats: - - "Tester Testerston " - - "Tester de Testerson" - """ - # Strip the email, if present - email = re.search("<.+>", name_str) - if email: - name_str = name_str.replace(email.group(), "") - - return name_str.strip().rsplit(" ", 1) - - -def generate_email(val: str) -> str: - """Takes a string and generates a placeholder email based on it""" - slug = slugify(val) - local_email = slug.replace("-", "_")[:64] - return f"{local_email}@example.com" - def parse_date(date_str): """Parses a date string to a datetime. Does not return an error.""" From 0bbbfd708f9f8186b1ed20239bdbe33138a55b26 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 10 Mar 2023 10:04:31 -0800 Subject: [PATCH 14/16] :fire: More dead code --- users/migrations/0008_auto_20230303_1843.py | 27 --------------------- users/migrations/0009_auto_20230303_2102.py | 25 ------------------- 2 files changed, 52 deletions(-) delete mode 100644 users/migrations/0008_auto_20230303_1843.py delete mode 100644 users/migrations/0009_auto_20230303_2102.py diff --git a/users/migrations/0008_auto_20230303_1843.py b/users/migrations/0008_auto_20230303_1843.py deleted file mode 100644 index 0f116a8f..00000000 --- a/users/migrations/0008_auto_20230303_1843.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 3.2.2 on 2023-03-03 18:43 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("users", "0007_user_image"), - ] - - operations = [ - migrations.AddField( - model_name="user", - name="claimed", - field=models.BooleanField( - default=True, verbose_name="whether this account has been claimed" - ), - ), - migrations.AddField( - model_name="user", - name="valid_email", - field=models.BooleanField( - default=True, verbose_name="whether the user's email address is valid" - ), - ), - ] diff --git a/users/migrations/0009_auto_20230303_2102.py b/users/migrations/0009_auto_20230303_2102.py deleted file mode 100644 index fbc72a3f..00000000 --- a/users/migrations/0009_auto_20230303_2102.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 3.2.2 on 2023-03-03 21:02 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("users", "0008_auto_20230303_1843"), - ] - - operations = [ - migrations.AlterField( - model_name="user", - name="claimed", - field=models.BooleanField( - default=True, verbose_name="Account has been claimed" - ), - ), - migrations.AlterField( - model_name="user", - name="valid_email", - field=models.BooleanField(default=True, verbose_name="Valid email address"), - ), - ] From be1d7d7c8d95de91a624be763bc94822bbc1735c Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 10 Mar 2023 11:14:16 -0800 Subject: [PATCH 15/16] :fire: Dead code --- .../commands/create_authors_maintainers.py | 154 ------------------ 1 file changed, 154 deletions(-) delete mode 100644 libraries/management/commands/create_authors_maintainers.py diff --git a/libraries/management/commands/create_authors_maintainers.py b/libraries/management/commands/create_authors_maintainers.py deleted file mode 100644 index a73003f6..00000000 --- a/libraries/management/commands/create_authors_maintainers.py +++ /dev/null @@ -1,154 +0,0 @@ -import djclick as click -import random -import re -import structlog - -from datetime import timedelta -from django.contrib.auth import get_user_model -from django.utils import timezone -from django.utils.text import slugify -from faker import Faker - -from libraries import utils -from libraries.github import LibraryUpdater -from libraries.models import Library, LibraryVersion -from versions.models import Version - -fake = Faker() - -logger = structlog.get_logger() - -User = get_user_model() - - -@click.command() -def command(): - """ - Idempotent. - - Get all the libraries. - - For each library: - - Retrieve and extract info about their authors and maintainers - - Use that info to create or update User records - - Associate the User records for those authors and maintainers to the - Library - - - [x] Retrieve their GH record - - [x] Get the file with the data in it and parse it - - [x] Extract author and maintainer data - - [x] Method to extract email - - [x] Method to extract first and last - - [x] If no email, method to create fake email - - [x] Add a User model field to denote a fake email - - [x] Add a User model field to denote unclaimed user - - [x] Method to get or create new user with email - - [x] If created, mark as Unclaimed - - [x] If fake email, mark fake email - - [x] Add the authors as Authors to the Library - - # Dealing with maintainers - - [x] Remove 'maintainers' from Library and add to LibraryVersion - - [x] Retrieve most recent LibraryVersion - - [x] Add maintainers to the most recent LibraryVersion - - NEW ISSUES - - [ ] Process to claim your user with your email - - [ ] Add process to update authors to the library syncing - - [ ] Add process to update maintainers to the LibraryVersion syncing - """ - - library = Library.objects.order_by("?").first() - version = Version.objects.most_recent() - click.secho(f"Getting Library data for '{library.name}'...", fg="green") - - updater = LibraryUpdater() - result = updater.get_library_metadata(library.name) - - if type(result) is list: - breakpoint() - # TODO: See line 211 in github.py - raise - - authors = result.get("authors") - maintainers = result.get("maintainers") - - click.secho(f"Getting authors...", fg="green") - for a in authors: - person_data = get_person_data(a) - user, created = User.objects.get_or_create( - email=person_data["email"], - defaults={ - "first_name": person_data["first_name"][:30], - "last_name": person_data["last_name"][:30], - }, - ) - click.secho(f"User {user.email} saved. Created? {created}", fg="green") - - if created: - user.claimed = False - user.is_active = False - user.save() - - library.authors.add(user) - click.secho( - f"User {user.email} added as a maintainer of {library.name}", fg="green" - ) - - if maintainers: - try: - library_version = LibraryVersion.objects.get( - library=library, version=version - ) - except LibraryVersion.DoesNotExist: - logger.info("No library version", version=version.pk, library=library.pk) - return - - click.secho(f"Getting maintainers...", fg="green") - for m in maintainers: - person_data = get_person_data(m) - - user, created = User.objects.get_or_create( - email=person_data["email"], - defaults={ - "first_name": person_data["first_name"][:30], - "last_name": person_data["last_name"][:30], - }, - ) - click.secho(f"--User {user.email} saved. Created? {created}", fg="green") - - if created: - user.claimed = False - user.is_active = False - user.valid_email = person_data["valid_email"] - user.save() - - library_version.maintainers.add(user) - click.secho( - f"--User {user.email} added as a maintainer of {library.name}", - fg="green", - ) - - click.secho("All done!", fg="green") - - -def get_person_data(person: str) -> dict: - """Takes an author/maintainer string and returns a dict with their data""" - person_data = {} - person_data["meta"] = person - person_data["valid_email"] = False - - names = utils.get_names(person) - person_data["first_name"] = names[0] - person_data["last_name"] = names[1] - - email = utils.extract_email(person) - if email: - person_data["email"] = email - person_data["valid_email"] = True - else: - person_data["email"] = utils.generate_email( - f"{person_data['first_name']} {person_data['last_name']}" - ) - - return person_data From 56e71df5ba0e8504c1fd528dd3125f8ba9f5e738 Mon Sep 17 00:00:00 2001 From: Lacey Williams Henschel Date: Fri, 10 Mar 2023 11:39:05 -0800 Subject: [PATCH 16/16] :pencil: Add template for library_version due to maintainers M2M --- templates/users/_library_version.html | 10 ++++++++++ templates/users/profile.html | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 templates/users/_library_version.html diff --git a/templates/users/_library_version.html b/templates/users/_library_version.html new file mode 100644 index 00000000..40d7445e --- /dev/null +++ b/templates/users/_library_version.html @@ -0,0 +1,10 @@ +
+

+ + {{ library_version.library.name }} + +

+

+ {{ library_version.library.description }} +

+
diff --git a/templates/users/profile.html b/templates/users/profile.html index 407f0f92..73d009fa 100644 --- a/templates/users/profile.html +++ b/templates/users/profile.html @@ -135,8 +135,8 @@

Libraries Maintained

- {% for library in maintained %} - {% include 'users/_library.html' with library=library %} + {% for library_version in maintained %} + {% include 'users/_library_version.html' with library_version=library_version %} {% empty %}

No Libraries Authored

{% endfor %}