website-v2/libraries/management/commands/release_tasks.py

import traceback
from contextlib import suppress
from datetime import timedelta

import djclick as click

from django.utils import timezone
from django.core.management import call_command
from django.contrib.auth import get_user_model
from django.conf import settings
from slack_sdk.errors import SlackApiError

from core.githubhelper import GithubAPIClient
from core.management.actions import (
    progress_message,
    Action,
    ActionsManager,
    send_notification,
)
from libraries.tasks import update_commits, generate_release_report
from reports.models import WebsiteStatReport
from slack.management.commands.fetch_slack_activity import get_my_channels, locked
from versions.models import Version, ReportConfiguration

User = get_user_model()


class ReleaseTasksManager(ActionsManager):
    latest_version: Version | None = None
    handled_commits: dict[str, int] = {}

    def __init__(
        self, base_uri: str, user_id: int, should_generate_report: bool = False
    ):
        self.base_uri = base_uri
        self.should_generate_report = should_generate_report
        self.user_id = user_id
        super().__init__()

    def set_tasks(self):
        self.tasks = [
            Action("Importing versions", self.import_versions),
            Action(
                "Importing most recent beta version",
                ["import_beta_release", "--delete-versions"],
            ),
            Action("Importing development versions", ["import_development_versions"]),
            Action("Importing libraries", ["update_libraries"]),
            Action(
                "Saving library-version relationships", self.import_library_versions
            ),
            Action("Adding library maintainers", ["update_maintainers"]),
            Action("Adding library authors", ["update_authors"]),
            Action(
                "Adding library version authors", ["update_library_version_authors"]
            ),
            Action("Importing git commits", self.import_commits),
            Action("Syncing mailinglist statistics", ["sync_mailinglist_stats"]),
            Action("Updating github issues", ["update_issues"]),
            Action("Updating slack activity buckets", ["fetch_slack_activity"]),
            Action("Updating website statistics", self.update_website_statistics),
            Action("Importing mailing list counts", self.import_ml_counts),
            Action("Generating report", self.generate_report),
        ]

    def import_versions(self):
        call_command("import_versions")
        self.latest_version = Version.objects.with_partials().most_recent()

    def import_library_versions(self):
        latest_version_number = self.latest_version.name.lstrip("boost-")
        call_command("import_library_versions", min_release=latest_version_number)

    def import_commits(self):
        self.handled_commits = update_commits(min_version=self.latest_version.name)

    def update_website_statistics(self):
        report, _ = WebsiteStatReport.objects.get_or_create(version=self.latest_version)
        report.populate_from_api()

    def import_ml_counts(self):
        """Import counts for the last four months. Should be more than enough,
        and saves lots of time vs importing all.
        """
        start_date = timezone.now() - timedelta(days=120)
        date_string = start_date.strftime("%Y-%m-%d")
        call_command("import_ml_counts", start_date=date_string)

    def generate_report(self):
        if not self.should_generate_report:
            self.add_progress_message("Skipped - report generation not requested")
            return

        report_configuration = ReportConfiguration.objects.get(
            version=self.latest_version.name
        )
        generate_release_report.delay(
            user_id=self.user_id,
            params={"report_configuration": report_configuration.id, "publish": True},
            base_uri=self.base_uri,
        )


@locked(1138692)
def run_commands(
    progress: list[str], base_uri: str, user_id: int, generate_report: bool = False
):
    manager = ReleaseTasksManager(
        base_uri=base_uri,
        should_generate_report=generate_report,
        user_id=user_id,
    )
    manager.run_tasks()
    progress.extend(manager.progress_messages)
    return manager.handled_commits


def bad_credentials() -> list[str]:
    """This management command requires access to Slack and GitHub APIs.
    Checks that credentials are available and valid.

    Returns a list of credentials that are invalid or missing.

    Good return: []
    Bad return: ["GITHUB_TOKEN", "SLACK_BOT_TOKEN"]
    """
    possibly_bad_credentials = ["GITHUB_TOKEN", "SLACK_BOT_TOKEN"]
    if settings.GITHUB_TOKEN:
        client = GithubAPIClient(settings.GITHUB_TOKEN)
        if client.is_authenticated():
            # If this is true, the GitHub token is good
            possibly_bad_credentials.remove("GITHUB_TOKEN")

    if settings.SLACK_BOT_TOKEN:
        with suppress(SlackApiError):  # just breaks on this error
            next(get_my_channels())
            # If we get this far, the Slack token is good
            possibly_bad_credentials.remove("SLACK_BOT_TOKEN")

    return possibly_bad_credentials


@click.command()
@click.option(
    "--base_uri",
    is_flag=False,
    help="The URI to which paths should be relative",
    default=None,
)
@click.option(
    "--user_id",
    is_flag=False,
    help="The ID of the user that started this task (For notification purposes)",
)
@click.option(
    "--generate_report",
    is_flag=True,
    help="Generate a report at the end of the command",
    default=False,
)
def command(user_id, base_uri=None, generate_report=False):
    """A long running chain of tasks to import and update library data."""
    start = timezone.now()

    user = User.objects.filter(id=user_id).first()

    progress = ["___Progress Messages___"]
    if missing_creds := bad_credentials():
        progress.append(
            progress_message(f"Missing credentials {', '.join(missing_creds)}")
        )
        send_notification(
            user,
            message="Your task `release_tasks` failed.",
            subject="Task Failed: release_tasks",
        )
        return

    send_notification(
        user,
        f"Your task `release_tasks` was started at: {start}",
        subject="Task Started: release_tasks",
    )

    try:
        handled_commits = run_commands(progress, base_uri, generate_report, user_id)
        end = timezone.now()
    except Exception:
        error = traceback.format_exc()
        message = [
            f"ERROR: There was an error while running release_tasks.\n\n{error}",
            "\n".join(progress),
        ]
        send_notification(
            user, "\n\n".join(message), subject="Task Failed: release_tasks"
        )
        raise

    zero_commit_libraries = [
        (key, val) for key, val in handled_commits.items() if val == 0
    ]
    message = [
        f"The task `release_tasks` was completed. Task took: {end - start}",
        "\n".join(progress),
    ]
    if zero_commit_libraries:
        zero_commit_message = [
            "The import_commits task did not find commits for these libraries.",
            "The task may need to re-run.",
        ]
        for lib, _ in zero_commit_libraries:
            zero_commit_message.append(lib)
        message.append("\n".join(zero_commit_message))
    send_notification(
        user,
        "\n\n".join(message),
        subject="Task Complete: release_tasks",
    )