From c6335c2fe2c7188f4cce10b3abf558a2eb62a656 Mon Sep 17 00:00:00 2001 From: Dave O'Connor Date: Mon, 18 Aug 2025 11:47:06 -0700 Subject: [PATCH] Updated import commands to allow and default to 'new' functionality. Remove artifactory-related commands. (#1744) --- docs/commands.md | 56 ++++++------------- docs/release_downloads.md | 12 ---- .../import_library_version_docs_urls.py | 41 +++++++++----- .../commands/import_library_versions.py | 26 ++++++--- .../management/commands/release_tasks.py | 2 +- .../commands/import_archives_release_data.py | 18 +++++- .../import_artifactory_release_data.py | 50 ----------------- .../commands/import_release_notes.py | 5 +- .../management/commands/import_versions.py | 2 +- .../migrations/0023_version_fully_imported.py | 21 +++++++ versions/models.py | 5 +- versions/tasks.py | 21 ++++++- 12 files changed, 125 insertions(+), 134 deletions(-) delete mode 100644 docs/release_downloads.md delete mode 100644 versions/management/commands/import_artifactory_release_data.py create mode 100644 versions/migrations/0023_version_fully_imported.py diff --git a/docs/commands.md b/docs/commands.md index 88a46c29..3ccb7edd 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -4,7 +4,6 @@ - [`boost_setup`](#boost_setup) - [`import_versions`](#import_versions) - [`import_archives_release_data`](#import_archives_release_data) - - [`import_artifactory_release_data`](#import_artifactory_release_data) - [`update_libraries`](#update_libraries) - [`import_library_versions`](#import_library_versions) - [`import_library_version_docs_urls`](#import_library_version_docs_urls) @@ -49,16 +48,16 @@ Imports `Version` objects from GitHub. **Options** -| Options | Format | Description | -|----------------------|--------|--------------------------------------------------------------| -| `--delete-versions` | bool | If passed, will delete all Version objects before importing Versions. | -| `--new` | bool | If passed, will import only new Version objects. | -| `--token` | string | GitHub API Token. If passed, will use this value. If not passed, will use the value in settings. | +| Options | Format | Description | +|----------------------|--------|---------------------------------------------------------------------------------------------------------| +| `--delete-versions` | bool | If passed, will delete all Version objects before importing Versions. | +| `--new` | bool | Default: 'true'. If 'true', will import only new Version objects. Set to 'false' to import all versions | +| `--token` | string | GitHub API Token. If passed, will use this value. If not passed, will use the value in settings. | **Process** - Retrieves the tags for the GitHub repo in `BASE_GITHUB_URL` -- Loops through all tags, and discards any that do not match our inclusion logic +- Loops through all tags, and discards any that do not match our inclusion logic, by default only versions that haven't already been imported. - For each successful tag, import it as a `Version` object - Then, run the command to the release downloads from Artifactory as `VersionFile` objects @@ -76,40 +75,16 @@ Import `VersionFile` objects from Artifactory. **Options** -| Options | Format | Description | -|----------------------|--------|--------------------------------------------------------------| -| `--release` | string | Format: `boost-1.63.0`. If passed, will import Archive urls for only that version. | +| Options | Format | Description | +|------------|--------|------------------------------------------------------------------------------------------------------------------------------| +| `--new` | bool | Default: 'true'. If 'true', will import only the newest release data. Set to 'false' to import archive data for all releases | +| `--release` | string | Format: `boost-1.63.0`. If passed, will import Archive urls for only that release. Overrides --new | **More Information** -- Loops through `Version` objects and calls the task that retrieves the Archives data with the version information +- Loops through `Version` objects, by default only the most recent one, and calls the task that retrieves the Archives data with the version information - Saves the Archives JSON data as `VersionFile` objects - -## `import_artifactory_release_data` - -*This process was run automatically as part of `import_versions`, but has been replaced by `import_archives_release_data`.* - -Import `VersionFile` objects from Artifactory. - -**Example** - -```bash -./manage.py import_artifactory_release_data -``` - -**Options** - -| Options | Format | Description | -|----------------------|--------|--------------------------------------------------------------| -| `--release` | string | Format: `boost-1.63.0`. If passed, will import Artifactory urls for only that version. | - -**More Information** - -- Loops through `Version` objects and calls the task that retrieves the Artifactory data with the version information -- Saves the Artifactory data as `VersionFile` objects - - ## `update_libraries` **Purpose**: Import and update `Library` and `Category` objects. Runs the library update script, which cycles through the repos listed in the Boost library and syncs their information. Most library information comes from `meta/libraries.json` stored in each Boost library repo. @@ -138,14 +113,15 @@ Import `VersionFile` objects from Artifactory. **Options** -| Options | Format | Description | -|----------------------|--------|--------------------------------------------------------------| -| `--token` | string | GitHub API Token. If passed, will use this value. If not passed, will use the value in settings. | +| Options | Format | Description | +|----------------------|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--token` | string | GitHub API Token. If passed, will use this value. If not passed, will use the value in settings. | | `--release` | string | Format: `boost-1.63.0`. If passed, will import Artifactory urls for only that version. Partial versions are accepted (so "1.7" will import libraries for version 1.70.0, 1.71.0, etc.) | +| `--new` | bool | Default: 'true'. If 'true', will import data for the newest release. Set to 'false' to import library version data for all releases | **Process** -- Loops through `Version` objects based on passed-in options +- Loops through `Version` objects based on passed-in options, by default just the most recent one. - For each `Version`, gets the libraries in that release from the `.gitmodules` file using the GitHub API - For each library listed in the `.gitmodules` file, get the complete list of libraries from the library's `meta/libraries.json` file (in its GitHub repo) using the GitHub API. (A single library repo might contain information for multiple libraries. Example: Functional also hosts Functional/Factory). - Save the `LibraryVersion` objects diff --git a/docs/release_downloads.md b/docs/release_downloads.md deleted file mode 100644 index d79b4c12..00000000 --- a/docs/release_downloads.md +++ /dev/null @@ -1,12 +0,0 @@ -# Boost Release Downloads - -## Artifactory - -- Populate the release downloads by running `./manage.py import_artifactory_release_data`. See [Management Commands](./commands.md#import_artifactory_release_data) for more information. -- Downloads for new versions populate as part of the new version import process -- Existing data can be refreshed by running `./manage.py import_artifactory_release_data` in the desired environment -- Environment variables: `ARTIFACTORY_URL` and `MIN_ARTIFACTORY_RELEASE`. See the [Envrionment Variables](./env_vars.md) for more information. - -The Artifactory API URL in `ARTIFACTORY_URL` allows us to retrieve the data about the downloads from the Artifactory API. - -The URLs for the downloads themselves are retrieved from the URLs provided to us by the Artifactory API. We don't generate the download links ourselves. diff --git a/libraries/management/commands/import_library_version_docs_urls.py b/libraries/management/commands/import_library_version_docs_urls.py index cc869831..7321a760 100644 --- a/libraries/management/commands/import_library_version_docs_urls.py +++ b/libraries/management/commands/import_library_version_docs_urls.py @@ -9,17 +9,24 @@ from versions.models import Version @click.command() @click.option( - "--version", + "--release", help="Boost version number (example: 1.81.0). If a partial version number is " "provided, the command process all versions that contain the partial version " "number (example: '--version='1.7' would process 1.7.0, 1.7.1, 1.7.2, etc.)", ) +@click.option( + "--new", + default=True, + type=click.BOOL, + help="True (default) Import library docs for newest version only from the db. False" + " downloads docs for all versions", +) @click.option( "--min-version", default=settings.MINIMUM_BOOST_VERSION, help="Minimum Boost version to process (default: 1.30.0)", ) -def command(version: str, min_version: str): +def command(release: str, new: bool, min_version: str): """Cycles through all Versions in the database, and for each version gets the corresponding tag's .gitmodules. @@ -28,40 +35,44 @@ def command(version: str, min_version: str): add maintainers to LibraryVersions. Args: - version (str): Boost version number (example: 1.81.0). If a partial version + release (str): Boost version number (example: 1.81.0). If a partial version number is provided, the command process all versions that contain the partial version number (example: "--version="1.7" would process 1.70.0, 1.70.1, 1.71.0, etc.) + new (bool): If True (default), only imports library docs for the most recent + version, if False it processes all versions greater than or equal to + min_version. Overridden by --release if provided. min_version (str): Minimum Boost version to process. If this is passed, then only versions that are greater than or equal to this version will be processed. """ click.secho("Saving links to version-specific library docs...", fg="green") min_version = f"boost-{min_version}" - if version is None: - versions = Version.objects.active().filter(name__gte=min_version) + version_qs = Version.objects.active().filter(name__gte=min_version) + if release: + versions = Version.objects.filter(name__icontains=release).order_by("-name") + elif new: + versions = [Version.objects.most_recent()] else: - versions = Version.objects.filter( - name__icontains=version, name__gte=min_version - ) + versions = version_qs.order_by("-name") # For each version, get the library version documentation url paths - for version in versions.order_by("-name"): - click.echo(f"Processing version {version.name}...") + for release in versions: + click.echo(f"Processing version {release.name}...") try: - get_and_store_library_version_documentation_urls_for_version(version.pk) + get_and_store_library_version_documentation_urls_for_version(release.pk) except ValueError as e: click.secho(e, fg="red") continue - for version in versions.order_by("-name"): - library_versions = LibraryVersion.objects.filter(version=version) - click.secho(f"Processing version {version.name}...", fg="green") + for release in versions: + library_versions = LibraryVersion.objects.filter(version=release) + click.secho(f"Processing version {release.name}...", fg="green") for library_version in library_versions: if not library_version.documentation_url: click.secho( f"Could not get docs url for {library_version.library.name} " - f"({version.name}).", + f"({release.name}).", fg="red", ) continue diff --git a/libraries/management/commands/import_library_versions.py b/libraries/management/commands/import_library_versions.py index c679326e..3987ceaf 100644 --- a/libraries/management/commands/import_library_versions.py +++ b/libraries/management/commands/import_library_versions.py @@ -9,12 +9,20 @@ from versions.tasks import import_library_versions @click.command() @click.option("--token", help="Github API token") @click.option("--release", help="Boost version number (example: 1.81.0)") +@click.option( + "--new", + default=True, + type=click.BOOL, + help="Update only the library versions for the most recent version (default: True)." + " False processes library versions for all versions greater than or equal to" + " min-release.", +) @click.option( "--min-release", default=settings.MINIMUM_BOOST_VERSION, help="Minimum Boost version to process (default: 1.31.0)", ) -def command(min_release: str, release: str, token: str): +def command(min_release: str, release: str, new: bool, token: str): """Cycles through all Versions in the database, and for each version gets the corresponding tag's .gitmodules. @@ -30,18 +38,22 @@ def command(min_release: str, release: str, token: str): number is provided, the command process all versions that contain the partial version number (example: "--version="1.7" would process 1.7.0, 1.7.1, 1.7.2, etc.) + new (bool): If True (default), only imports library versions for the most recent + version, if False it processes all versions greater than or equal to min_release + Overridden by --release if provided. """ click.secho("Saving library-version relationships...", fg="green") min_release = f"boost-{min_release}" - if release is None: - versions = Version.objects.active().filter(name__gte=min_release) + versions_qs = Version.objects.active().filter(name__gte=min_release) + if release: + versions = versions_qs.filter(name__icontains=release).order_by("-name") + elif new: + versions = [Version.objects.most_recent()] else: - versions = Version.objects.filter( - name__icontains=release, name__gte=min_release - ) + versions = versions_qs.order_by("-name") - for version in versions.order_by("-name"): + for version in versions: version_type = "branch" if version.slug in settings.BOOST_BRANCHES else "tag" click.secho(f"Saving libraries for version {version.name}", fg="green") import_library_versions.delay( diff --git a/libraries/management/commands/release_tasks.py b/libraries/management/commands/release_tasks.py index 326c565f..a62ce0fe 100644 --- a/libraries/management/commands/release_tasks.py +++ b/libraries/management/commands/release_tasks.py @@ -100,7 +100,7 @@ class ReleaseTasksManager: return self.handled_commits def import_versions(self): - call_command("import_versions", "--new") + call_command("import_versions") self.latest_version = Version.objects.most_recent() def import_library_versions(self): diff --git a/versions/management/commands/import_archives_release_data.py b/versions/management/commands/import_archives_release_data.py index 3462c8a4..efdb9c21 100644 --- a/versions/management/commands/import_archives_release_data.py +++ b/versions/management/commands/import_archives_release_data.py @@ -13,26 +13,40 @@ from versions.releases import ( get_binary_checksums, get_binary_download_uris_for_release, ) +from structlog import get_logger + +logger = get_logger(__name__) @click.command() @click.option("--release", is_flag=False, help="Release name") -def command(release): +@click.option( + "--new", + default=True, + type=click.BOOL, + help="Choose the newest version only from the db. false downloads all", +) +def command(release: str, new: bool): """ Import release data from the Boost artifactory. This command will import the release data from the Boost artifactory for the specified release. If no release is specified, it will import the release data - for all releases included in Artifactory. + for the most recent release included in Artifactory. If --new is set to False, + it will import the release data for all releases that are greater than or equal to + the minimum release defined in settings.MIN_ARCHIVES_RELEASE. """ last_release = settings.MIN_ARCHIVES_RELEASE if release: versions = Version.objects.filter(name__icontains=release) + elif new: + versions = [Version.objects.most_recent()] else: versions = Version.objects.filter(name__gte=last_release) for v in versions: + logger.info(f"Processing release {v.name}") version_num = v.name.replace("boost-", "") try: archives_urls = get_archives_download_uris_for_release(version_num) diff --git a/versions/management/commands/import_artifactory_release_data.py b/versions/management/commands/import_artifactory_release_data.py deleted file mode 100644 index b013cf37..00000000 --- a/versions/management/commands/import_artifactory_release_data.py +++ /dev/null @@ -1,50 +0,0 @@ -import djclick as click - -import requests - -from django.conf import settings - -from versions.models import Version -from versions.releases import ( - get_artifactory_download_data, - get_artifactory_download_uris_for_release, - store_release_downloads_for_version, -) - - -@click.command() -@click.option("--release", is_flag=False, help="Release name") -def command(release): - """ - Import release data from the Boost artifactory. - - This command will import the release data from the Boost artifactory for the - specified release. If no release is specified, it will import the release data - for all releases included in Artifactory. - """ - last_release = settings.MIN_ARTIFACTORY_RELEASE - - if release: - versions = Version.objects.filter(name__icontains=release) - else: - versions = Version.objects.filter(name__gte=last_release) - - for v in versions: - version_num = v.name.replace("boost-", "") - try: - artifactory_data = get_artifactory_download_uris_for_release(version_num) - except requests.exceptions.HTTPError: - print(f"Skipping {version_num}, error retrieving release data") - continue - - download_data = [] - for d in artifactory_data: - try: - data = get_artifactory_download_data(d) - download_data.append(data) - except requests.exceptions.HTTPError: - print(f"Skipping {d}, error retrieving download data") - continue - - store_release_downloads_for_version(v, download_data) - print(f"Stored download data for {v.name}") diff --git a/versions/management/commands/import_release_notes.py b/versions/management/commands/import_release_notes.py index 94ecd358..32edb2c4 100644 --- a/versions/management/commands/import_release_notes.py +++ b/versions/management/commands/import_release_notes.py @@ -4,10 +4,11 @@ from versions.tasks import import_release_notes @click.command() -def command(): +@click.option("--new", default=True, help="Only import notes for new versions") +def command(new: bool): """ Import and process release notes for all available versions. """ click.secho("Importing release notes...", fg="green") - import_release_notes.delay() + import_release_notes.delay(new_versions_only=new) diff --git a/versions/management/commands/import_versions.py b/versions/management/commands/import_versions.py index 09086609..975f37de 100644 --- a/versions/management/commands/import_versions.py +++ b/versions/management/commands/import_versions.py @@ -5,7 +5,7 @@ from versions.tasks import import_versions @click.command() @click.option("--delete-versions", is_flag=True, help="Delete all existing versions") -@click.option("--new", is_flag=True, help="Only import new versions") +@click.option("--new", default=True, help="Only import new versions") @click.option("--token", is_flag=False, help="Github API token") def command( delete_versions, diff --git a/versions/migrations/0023_version_fully_imported.py b/versions/migrations/0023_version_fully_imported.py new file mode 100644 index 00000000..a47ecc23 --- /dev/null +++ b/versions/migrations/0023_version_fully_imported.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.16 on 2025-08-12 21:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("versions", "0022_alter_reportconfiguration_version"), + ] + + operations = [ + migrations.AddField( + model_name="version", + name="fully_imported", + field=models.BooleanField( + default=False, + help_text="Whether this version has been fully imported and is ready for use.", + ), + ), + ] diff --git a/versions/models.py b/versions/models.py index a17058d7..f149810f 100755 --- a/versions/models.py +++ b/versions/models.py @@ -39,7 +39,10 @@ class Version(models.Model): "beta release or a development version", ) data = models.JSONField(default=dict) - + fully_imported = models.BooleanField( + default=False, + help_text="Whether this version has been fully imported and is ready for use.", + ) objects = VersionManager() def __str__(self): diff --git a/versions/tasks.py b/versions/tasks.py index 89099dca..b306cdaa 100644 --- a/versions/tasks.py +++ b/versions/tasks.py @@ -50,6 +50,9 @@ def import_versions( Version.objects.all().delete() logger.info("import_versions_deleted_all_versions") + # delete any versions that were only partially imported so they are re-imported + Version.objects.filter(fully_imported=False).delete() + # Get all Boost tags from Github client = GithubAPIClient(token=token) tags = client.get_tags() @@ -61,7 +64,7 @@ def import_versions( if skip_tag(name, new_versions_only): continue - logger.info("import_versions_importing_version", version_name=name) + logger.info(f"import_versions importing version {name=}") import_version_task_group.append(import_version.s(name, tag=tag, token=token)) if import_version_task_group: @@ -70,15 +73,20 @@ def import_versions( if purge_after: logger.info("linking fastly purge") task_group.link(purge_fastly_release_cache.s()) + task_group.link(mark_fully_completed.s()) task_group() import_release_notes.delay() @app.task -def import_release_notes(): +def import_release_notes(new_versions_only=True): """Imports release notes from the existing rendered release notes in the repository.""" - for version in Version.objects.exclude(name__in=["master", "develop"]).active(): + versions = [Version.objects.most_recent()] + if not new_versions_only: + versions = Version.objects.exclude(name__in=["master", "develop"]).active() + + for version in versions: store_release_notes_task.delay(str(version.pk)) store_release_notes_in_progress_task.delay() @@ -486,6 +494,13 @@ def purge_fastly_release_cache(): logger.info(f"Sent fastly purge request for {service=}.") +@app.task +def mark_fully_completed(): + """Marks all versions as fully imported""" + Version.objects.filter(fully_imported=False).update(fully_imported=True) + logger.info("Marked all versions as fully imported.") + + # Helper functions