diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index d57b44ed2..fb9ed4a1c 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,34 +1,28 @@ -FROM {{ ci_base | default(pulp_default_container) }} +FROM {{ image.ci_base }} +{%- if image.webserver_snippet %} -# Add source directories to container -{% for item in plugins %} -ADD ./{{ item.name }} ./{{ item.name }} -{% endfor %} +ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf +{%- endif %} + +{%- for item in extra_files | default([]) %} -# Install python packages -# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads -# Hacking botocore (https://github.com/boto/botocore/pull/1990) +ADD ./{{ item.origin }} {{ item.destination }} +{%- endfor %} # This MUST be the ONLY call to pip install in inside the container. RUN pip3 install --upgrade pip setuptools wheel && \ rm -rf /root/.cache/pip && \ - pip3 install -{%- if s3_test | default(false) -%} -{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue -{%- endif -%} -{%- for item in plugins -%} -{{ " " }}{{ item.source }} -{%- if item.upperbounds | default(false) -%} -{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt + pip3 install {{ image.source }} +{%- if image.upperbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt {%- endif -%} -{%- if item.lowerbounds | default(false) -%} -{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt +{%- if image.lowerbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt {%- endif -%} -{%- if item.ci_requirements | default(false) -%} -{{ " " }}-r ./{{ item.name }}/ci_requirements.txt +{%- if image.ci_requirements | default(false) -%} +{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt {%- endif -%} -{%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ +{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \ rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} @@ -46,11 +40,8 @@ ENV {{ key | upper }}={{ value }} USER pulp:pulp RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link -USER root:root -{% for item in plugins %} -RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ - ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true -{% endfor %} +RUN mkdir /var/lib/pulp/.config +USER root:root ENTRYPOINT ["/init"] diff --git a/.ci/ansible/build_container.yaml b/.ci/ansible/build_container.yaml index c380b430a..0a188cba9 100644 --- a/.ci/ansible/build_container.yaml +++ b/.ci/ansible/build_container.yaml @@ -1,15 +1,14 @@ # Ansible playbook to create the pulp service containers image --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Generate Containerfile from template" - template: - src: Containerfile.j2 - dest: Containerfile - + ansible.builtin.template: + src: "Containerfile.j2" + dest: "Containerfile" - name: "Build pulp image" # We build from the ../.. (parent dir of pulpcore git repo) Docker build # "context" so that repos like pulp-smash are accessible to Docker diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 7a73ea2f3..dfe2851de 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -26,56 +26,3 @@ API_ROOT = {{ api_root | repr }} {% endfor %} {% endif %} -{% if s3_test | default(false) %} -MEDIA_ROOT: "" -S3_USE_SIGV4 = True -{% if test_storages_compat_layer is defined and test_storages_compat_layer %} -STORAGES = { - "default": { - "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", - "OPTIONS": { - "access_key": "{{ minio_access_key }}", - "secret_key": "{{ minio_secret_key }}", - "region_name": "eu-central-1", - "addressing_style": "path", - "signature_version": "s3v4", - "bucket_name": "pulp3", - "endpoint_url": "http://minio:9000", - "default_acl": "@none None", - }, - }, - "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", - }, -} -{% else %} -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" -AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" -AWS_S3_REGION_NAME = "eu-central-1" -AWS_S3_ADDRESSING_STYLE = "path" -AWS_S3_SIGNATURE_VERSION = "s3v4" -AWS_STORAGE_BUCKET_NAME = "pulp3" -AWS_S3_ENDPOINT_URL = "http://minio:9000" -AWS_DEFAULT_ACL = "@none None" -{% endif %} -{% endif %} - -{% if azure_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" -MEDIA_ROOT = "" -AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -AZURE_ACCOUNT_NAME = "devstoreaccount1" -AZURE_CONTAINER = "pulp-test" -AZURE_LOCATION = "pulp3" -AZURE_OVERWRITE_FILES = True -AZURE_URL_EXPIRATION_SECS = 120 -AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' -{% endif %} - -{% if gcp_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" -MEDIA_ROOT = "" -GS_BUCKET_NAME = "gcppulp" -GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443" -{% endif %} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 47e5221e5..acdc22ad3 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -1,33 +1,26 @@ # Ansible playbook to start the pulp service container and its supporting services --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Create Settings Directories" - file: + ansible.builtin.file: path: "{{ item }}" - state: directory + state: "directory" mode: "0755" loop: - - settings - - ssh - - ~/.config/pulp_smash + - "settings" - name: "Generate Pulp Settings" template: - src: settings.py.j2 - dest: settings/settings.py - - - name: "Configure pulp-smash" - copy: - src: smash-config.json - dest: ~/.config/pulp_smash/settings.json + src: "settings.py.j2" + dest: "settings/settings.py" - name: "Setup docker networking" docker_network: - name: pulp_ci_bridge + name: "pulp_ci_bridge" - name: "Start Service Containers" docker_container: @@ -37,24 +30,24 @@ recreate: true privileged: true networks: - - name: pulp_ci_bridge + - name: "pulp_ci_bridge" aliases: "{{ item.name }}" volumes: "{{ item.volumes | default(omit) }}" env: "{{ item.env | default(omit) }}" command: "{{ item.command | default(omit) }}" - state: started + state: "started" loop: "{{ services | default([]) }}" - name: "Retrieve Docker Network Info" docker_network_info: - name: pulp_ci_bridge - register: pulp_ci_bridge_info + name: "pulp_ci_bridge" + register: "pulp_ci_bridge_info" - name: "Update /etc/hosts" lineinfile: - path: /etc/hosts + path: "/etc/hosts" regexp: "\\s{{ item.value.Name }}\\s*$" - line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}" + line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}" loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}" become: true @@ -63,19 +56,19 @@ aws_access_key: "{{ minio_access_key }}" aws_secret_key: "{{ minio_secret_key }}" s3_url: "http://minio:9000" - region: eu-central-1 - name: pulp3 - state: present - when: s3_test | default(false) + region: "eu-central-1" + name: "pulp3" + state: "present" + when: "s3_test | default(false)" - block: - name: "Wait for Pulp" uri: url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/" - follow_redirects: all - validate_certs: no - register: result - until: result.status == 200 + follow_redirects: "all" + validate_certs: "no" + register: "result" + until: "result.status == 200" retries: 12 delay: 5 rescue: @@ -86,7 +79,7 @@ - name: "Check version of component being tested" assert: that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver) + - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)" fail_msg: | Component {{ item.app_label }} was expected to be installed in version {{ component_version }}. Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}. @@ -100,9 +93,20 @@ login admin password password -- hosts: pulp +- hosts: "pulp" gather_facts: false tasks: + - name: "Create directory for pulp-smash config" + ansible.builtin.file: + path: "/var/lib/pulp/.config/pulp_smash/" + state: "directory" + mode: "0755" + + - name: "Configure pulp-smash" + ansible.builtin.copy: + src: "smash-config.json" + dest: "/var/lib/pulp/.config/pulp_smash/settings.json" + - name: "Set pulp admin password" command: cmd: "pulpcore-manager reset-admin-password --password password" diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index 2617a4089..c9198f19a 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -1,7 +1,19 @@ # Pulpcore versions without the openapi command do no longer work in the CI -pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* - +# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70 +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.* tablib!=3.6.0 # 3.6.0: This release introduced a regression removing the "html" optional dependency. + + +multidict!=6.3.0 +# This release failed the lower bounds test for some case sensitivity in CIMultiDict. + + +azure-storage-blob!=12.28.* +# Apparently does not work with current azurite. + + +pycares<5 +# older aiodns versions don't pin pycares UB, and are broken by pycares>=5 diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 353151534..66c494e97 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -53,6 +53,9 @@ def to_upper_bound(req): if requirement.name == "pulpcore": # An exception to allow for pulpcore deprecation policy. return fetch_pulpcore_upper_bound(requirement) + # skip requirement with environment scopes. E.g 'foo==1.0.0;python_version>=3.9' + if requirement.marker: + return f"# ENVIRONMENT IS UNTRACKABLE: {req}" for spec in requirement.specifier: if spec.operator == "~=": return f"# NO BETTER CONSTRAINT: {req}" @@ -83,13 +86,15 @@ def to_lower_bound(req): else: for spec in requirement.specifier: if spec.operator == ">=": + min_version = spec.version if requirement.name == "pulpcore": # Currently an exception to allow for pulpcore bugfix releases. # TODO Semver libraries should be allowed too. operator = "~=" + if len(Version(min_version).release) != 3: + raise RuntimeError("Pulpcore lower bound must be in the form '>=x.y.z'.") else: operator = "==" - min_version = spec.version return f"{requirement.name}{operator}{min_version}" return f"# NO LOWER BOUND: {req}" diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index dfc4a41e3..611c882ca 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -1,20 +1,31 @@ #!/usr/bin/env python +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# "pyyaml>=6.0.3,<6.1.0", +# ] +# /// import argparse import re import os +import sys import tomllib -import yaml +import typing as t from pathlib import Path + +import yaml from packaging.version import Version from git import Repo RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" -Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"] -Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"] +Y_CHANGELOG_EXTS = [".feature"] +Z_CHANGELOG_EXTS = [".bugfix", ".misc"] -def options(): +def options() -> argparse.Namespace: """Check which branches need a release.""" parser = argparse.ArgumentParser() parser.add_argument( @@ -33,13 +44,13 @@ def options(): return parser.parse_args() -def template_config(): +def template_config() -> dict[str, t.Any]: # Assume this script lies in .ci/scripts path = Path(__file__).absolute().parent.parent.parent / "template_config.yml" return yaml.safe_load(path.read_text()) -def current_version(repo, commitish): +def current_version(repo: Repo, commitish: str) -> Version: try: pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml")) try: @@ -53,7 +64,7 @@ def current_version(repo, commitish): return Version(current_version) -def check_pyproject_dependencies(repo, from_commit, to_commit): +def check_pyproject_dependencies(repo: Repo, from_commit: str, to_commit: str) -> list[str]: try: new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml")) try: @@ -74,8 +85,8 @@ def check_pyproject_dependencies(repo, from_commit, to_commit): return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."] -def main(options, template_config): - DEFAULT_BRANCH = template_config["plugin_default_branch"] +def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int: + DEFAULT_BRANCH: str = template_config["plugin_default_branch"] repo = Repo() @@ -88,7 +99,7 @@ def main(options, template_config): # Warning: This will not work if branch names contain "/" but we don't really care here. heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")] - available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] + available_branches = [h for h in heads if re.fullmatch(RELEASE_BRANCH_REGEX, h)] available_branches.sort(key=lambda ver: Version(ver)) available_branches.append(DEFAULT_BRANCH) @@ -105,7 +116,10 @@ def main(options, template_config): if diff := branches - set(available_branches): print(f"Supplied branches contains non-existent branches! {diff}") - exit(1) + return 1 + + branches = [branch for branch in available_branches if branch in branches] + branches.reverse() print(f"Checking for releases on branches: {branches}") @@ -170,6 +184,8 @@ def main(options, template_config): if len(releases) == 0: print("No new releases to perform.") + return 0 + if __name__ == "__main__": - main(options(), template_config()) + sys.exit(main(options(), template_config())) diff --git a/.ci/scripts/clean_gh_release_notes.py b/.ci/scripts/clean_gh_release_notes.py new file mode 100755 index 000000000..2bf5f4c27 --- /dev/null +++ b/.ci/scripts/clean_gh_release_notes.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# This script is running with elevated privileges from the main branch against pull requests. +# +# It cleans the input from artifacts which are used by the pulp documentation internally, +# but clutter for GitHub releases + +import sys + +NOTE = """ +> [!NOTE] +> Official changes are available on [Pulp docs]({docs_url})\ +""" + + +def main(): + plugin_name = sys.argv[1] + version_str = sys.argv[2] + docs_url = f"https://pulpproject.org/{plugin_name}/changes/#{version_str}" + note_added = False + for line in sys.stdin: + if line.endswith("\n"): + line = line[:-1] + if line.startswith("#"): + print(line.split(" {: #")[0]) + if not note_added and version_str in line: + print(NOTE.format(docs_url=docs_url)) + note_added = True + else: + print(line) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 8cae45a96..fbb5d59d0 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -1,4 +1,12 @@ #!/bin/env python3 +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# ] +# /// + # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use @@ -7,16 +15,21 @@ # For more info visit https://github.com/pulp/plugin_template import itertools +import json import os import re import tomllib +import urllib.request +from pathlib import Path from git import GitCommandError, Repo from packaging.version import parse as parse_version + +PYPI_PROJECT = "pulp_python" + # Read Towncrier settings -with open("pyproject.toml", "rb") as fp: - tc_settings = tomllib.load(fp)["tool"]["towncrier"] +tc_settings = tomllib.loads(Path("pyproject.toml").read_text())["tool"]["towncrier"] CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( @@ -35,7 +48,7 @@ # see help(re.split) for more info. NAME_REGEX = r".*" VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*" -VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})" +VERSION_CAPTURE_REGEX = rf"(?:YANKED )?({VERSION_REGEX})" DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" TITLE_REGEX = ( "(" @@ -75,6 +88,20 @@ def main(): branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True) branches = [ref.name for ref in branches] + changed = False + + try: + response = urllib.request.urlopen(f"https://pypi.org/pypi/{PYPI_PROJECT}/json") + pypi_record = json.loads(response.read()) + yanked_versions = { + parse_version(version): release[0]["yanked_reason"] + for version, release in pypi_record["releases"].items() + if release[0]["yanked"] is True + } + except Exception: + # If something failed, just don't mark anything as yanked. + yanked_versions = {} + with open(CHANGELOG_FILE, "r") as f: main_changelog = f.read() preamble, main_changes = split_changelog(main_changelog) @@ -95,9 +122,19 @@ def main(): if left[0] != right[0]: main_changes.append(right) + if yanked_versions: + for change in main_changes: + if change[0] in yanked_versions and "YANKED" not in change[1].split("\n")[0]: + reason = yanked_versions[change[0]] + version = str(change[0]) + change[1] = change[1].replace(version, "YANKED " + version, count=1) + if reason: + change[1] = change[1].replace("\n", f"\n\nYank reason: {reason}\n", count=1) + changed = True + new_length = len(main_changes) - if old_length < new_length: - print(f"{new_length - old_length} new versions have been added.") + if old_length < new_length or changed: + print(f"{new_length - old_length} new versions have been added (or something has changed).") with open(CHANGELOG_FILE, "w") as fp: fp.write(preamble) for change in main_changes: diff --git a/.ci/scripts/skip_tests.py b/.ci/scripts/skip_tests.py new file mode 100755 index 000000000..a68d000d6 --- /dev/null +++ b/.ci/scripts/skip_tests.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +skip_tests.py - Check if only documentation files were changed in a git branch + +Usage: + ./skip_tests.py + +Arguments: + git_root: The root directory of the git project + reference_branch: The branch to compare against + +Returns: + 0: Skip + 1: NoSkip + *: Error +""" + +import sys +import os +import re +import git +import textwrap +import argparse + +DOC_PATTERNS = [ + r"^docs/", + r"\.md$", + r"LICENSE.*", + r"CHANGELOG.*", + r"CHANGES.*", + r"CONTRIBUTING.*", +] + +# Exit codes +CODE_SKIP = 0 +CODE_NO_SKIP = 1 +CODE_ERROR = 2 + + +def main() -> int: + git_root, reference_branch = get_args() + changed_files = get_changed_files(git_root, reference_branch) + if not changed_files: + return CODE_SKIP + doc_files = [f for f in changed_files if is_doc_file(f)] + not_doc_files = set(changed_files) - set(doc_files) + print_changes(doc_files, not_doc_files) + if not_doc_files: + return CODE_NO_SKIP + else: + return CODE_SKIP + + +# Utils + + +def get_changed_files(git_root: str, reference_branch: str) -> list[str]: + """Get list of files changed between current branch and reference branch.""" + repo = git.Repo(git_root) + diff_index = repo.git.diff("--name-only", reference_branch).strip() + if not diff_index: + return [] + return [f.strip() for f in diff_index.split("\n") if f.strip()] + + +def is_doc_file(file_path: str) -> bool: + """Check if a file is a documentation file.""" + for pattern in DOC_PATTERNS: + if re.search(pattern, file_path): + return True + return False + + +def print_changes(doc_files: list[str], not_doc_files: list[str]) -> None: + display_doc = " \n".join(doc_files) + print(f"doc_files({len(doc_files)})") + if doc_files: + display_doc = "\n".join(doc_files) + print(textwrap.indent(display_doc, " ")) + + print(f"non_doc_files({len(not_doc_files)})") + if not_doc_files: + display_non_doc = " \n".join(not_doc_files) + print(textwrap.indent(display_non_doc, " ")) + + +def get_args() -> tuple[str, str]: + """Parse command line arguments and validate them.""" + parser = argparse.ArgumentParser(description="Check if CI can skip tests for a git branch") + parser.add_argument("git_root", help="The root directory of the git project") + parser.add_argument("reference_branch", help="The branch to compare against") + args = parser.parse_args() + git_root = os.path.abspath(args.git_root) + ref_branch = args.reference_branch + + if not os.path.exists(git_root): + raise ValueError(f"Git root directory does not exist: {git_root}") + if not os.path.isdir(git_root): + raise ValueError(f"Git root is not a directory: {git_root}") + try: + git.Repo(git_root) + except git.InvalidGitRepositoryError: + raise ValueError(f"Directory is not a git repository: {git_root}") + return git_root, ref_branch + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + print(e) + sys.exit(CODE_ERROR) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py old mode 100755 new mode 100644 index bb9a7d8ba..a4dc9004a --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -1,66 +1,31 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template +# This file is managed by the plugin template. +# Do not edit. import os import re import subprocess import sys import tomllib +import yaml from pathlib import Path from github import Github -with open("pyproject.toml", "rb") as fp: - PYPROJECT_TOML = tomllib.load(fp) -KEYWORDS = ["fixes", "closes"] -BLOCKING_REGEX = [ - r"^DRAFT", - r"^WIP", - r"^NOMERGE", - r"^DO\s*NOT\s*MERGE", - r"^EXPERIMENT", - r"^FIXUP", - r"^fixup!", # This is created by 'git commit --fixup' - r"Apply suggestions from code review", # This usually comes from GitHub -] -try: - CHANGELOG_EXTS = [ - f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] - ] -except KeyError: - CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] -NOISSUE_MARKER = "[noissue]" - -sha = sys.argv[1] -message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - -if NOISSUE_MARKER in message: - sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") - -blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] -if blocking_matches: - print("Found these phrases in the commit message:") - for m in blocking_matches: - print(" - " + m.group(0)) - sys.exit("This PR is not ready for consumption.") -g = Github(os.environ.get("GITHUB_TOKEN")) -repo = g.get_repo("pulp/pulp_python") - - -def check_status(issue): +def check_status(issue, repo, cherry_pick): gi = repo.get_issue(int(issue)) if gi.pull_request: sys.exit(f"Error: issue #{issue} is a pull request.") - if gi.closed_at: + if gi.closed_at and not cherry_pick: + print("Make sure to use 'git cherry-pick -x' when backporting a change.") + print( + "If a backport of a change requires a significant amount of rewriting, " + "consider creating a new issue." + ) sys.exit(f"Error: issue #{issue} is closed.") -def check_changelog(issue): +def check_changelog(issue, CHANGELOG_EXTS): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -70,18 +35,63 @@ def check_changelog(issue): sys.exit(f"Invalid extension for changelog entry '{match}'.") -print("Checking commit message for {sha}.".format(sha=sha[0:7])) +def main() -> None: + TEMPLATE_CONFIG = yaml.safe_load(Path("template_config.yml").read_text()) + GITHUB_ORG = TEMPLATE_CONFIG["github_org"] + PLUGIN_NAME = TEMPLATE_CONFIG["plugin_name"] + + with Path("pyproject.toml").open("rb") as _fp: + PYPROJECT_TOML = tomllib.load(_fp) + KEYWORDS = ["fixes", "closes"] + BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"^fixup!", # This is created by 'git commit --fixup' + r"Apply suggestions from code review", # This usually comes from GitHub + ] + try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] + except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] + NOISSUE_MARKER = "[noissue]" + + sha = sys.argv[1] + message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") + + if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + + blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] + if blocking_matches: + print("Found these phrases in the commit message:") + for m in blocking_matches: + print(" - " + m.group(0)) + sys.exit("This PR is not ready for consumption.") + + g = Github(os.environ.get("GITHUB_TOKEN")) + repo = g.get_repo(f"{GITHUB_ORG}/{PLUGIN_NAME}") + + print("Checking commit message for {sha}.".format(sha=sha[0:7])) + + # validate the issue attached to the commit + issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords="|".join(KEYWORDS)) + issues = re.findall(issue_regex, message, re.IGNORECASE) + cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" + cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) + + if issues: + for issue in issues: + check_status(issue, repo, cherry_pick) + check_changelog(issue, CHANGELOG_EXTS) -# validate the issue attached to the commit -issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -issues = re.findall(issue_regex, message, re.IGNORECASE) -cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" -cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) + print("Commit message for {sha} passed.".format(sha=sha[0:7])) -if issues: - for issue in issues: - if not cherry_pick: - check_status(issue) - check_changelog(issue) -print("Commit message for {sha} passed.".format(sha=sha[0:7])) +if __name__ == "__main__": + main() diff --git a/.flake8 b/.flake8 index 64403dceb..9e00b3faa 100644 --- a/.flake8 +++ b/.flake8 @@ -8,13 +8,14 @@ exclude = ./docs/*,*/migrations/* per-file-ignores = */__init__.py: F401 -ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402 +ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 max-line-length = 100 # Flake8 builtin codes # -------------------- # E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed # W503: This enforces operators before line breaks which is not pep8 or black compatible. +# F824: 'nonlocal' is unused: name is never assigned in scope # Flake8-quotes extension codes # ----------------------------- diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..4bee60faf --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,15 @@ + + +### 📜 Checklist + +- [ ] Commits are cleanly separated with meaningful messages (simple features and bug fixes should be [squashed](https://pulpproject.org/pulpcore/docs/dev/guides/git/#rebasing-and-squashing) to one commit) +- [ ] A [changelog entry](https://pulpproject.org/pulpcore/docs/dev/guides/git/#changelog-update) or entries has been added for any significant changes +- [ ] Follows the [Pulp policy on AI Usage](https://pulpproject.org/help/more/governance/ai_policy/) +- [ ] (For new features) - User documentation and test coverage has been added + +See: [Pull Request Walkthrough](https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/) diff --git a/.github/template_gitref b/.github/template_gitref deleted file mode 100644 index 4630e6c46..000000000 --- a/.github/template_gitref +++ /dev/null @@ -1 +0,0 @@ -2021.08.26-421-g204a709 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 67878dc45..988902a0f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,23 +19,21 @@ jobs: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install build packaging twine wheel mkdocs jq - echo ::endgroup:: - name: "Build package" run: | python3 -m build @@ -48,7 +46,7 @@ jobs: pulpcore-manager openapi --file "api.json" pulpcore-manager openapi --bindings --component "python" --file "python-api.json" - name: "Upload Package whl" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "plugin_package" path: "pulp_python/dist/" @@ -56,7 +54,7 @@ jobs: retention-days: 5 overwrite: true - name: "Upload API specs" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "api_spec" path: | @@ -75,7 +73,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - name: "Upload python client packages" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "python-client.tar" path: | @@ -84,7 +82,7 @@ jobs: retention-days: 5 overwrite: true - name: "Upload python client docs" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "python-client-docs.tar" path: | @@ -102,7 +100,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - name: "Upload Ruby client" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "ruby-client.tar" path: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d82f9fee0..2965afca1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,18 +21,16 @@ jobs: check-commits: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install requests pygithub - echo ::endgroup:: + pip install requests pygithub pyyaml - name: "Check commit message" if: github.event_name == 'pull_request' env: @@ -43,14 +41,65 @@ jobs: run: | .github/workflows/scripts/check_commit.sh + check-changes: + runs-on: "ubuntu-latest" + outputs: + run_tests: "${{ steps.check.outputs.run_tests }}" + run_docs: "${{ steps.check.outputs.run_docs }}" + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 0 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.12" + + - name: "Install python dependencies" + run: | + pip install gitpython + + - name: "Analyze changed files" + id: "check" + shell: "bash" + run: | + # We only test docs on the default branch (usually main) + if [[ "${{ github.base_ref }}" == *"main" ]]; then + echo "run_docs=1" >> $GITHUB_OUTPUT + else + echo "run_docs=0" >> $GITHUB_OUTPUT + fi + + set +e + BASE_REF=${{ github.event.pull_request.base.sha }} + echo "Checking against:" + git name-rev $BASE_REF + python3 .ci/scripts/skip_tests.py . $BASE_REF + exit_code=$? + if [ $exit_code -ne 0 ] && [ $exit_code -ne 1 ]; then + echo "Error: skip_tests.py returned unexpected exit code $exit_code" + exit $exit_code + fi + echo "run_tests=$exit_code" >> $GITHUB_OUTPUT + docs: + needs: "check-changes" uses: "./.github/workflows/docs.yml" + with: + run_docs: "${{ needs.check-changes.outputs.run_docs }}" lint: uses: "./.github/workflows/lint.yml" + sanity: + uses: "./.github/workflows/sanity.yml" + build: - needs: "lint" + needs: + - "check-changes" + - "lint" + if: "needs.check-changes.outputs.run_tests == '1'" uses: "./.github/workflows/build.yml" test: @@ -62,7 +111,7 @@ jobs: deprecations: runs-on: "ubuntu-latest" - if: github.base_ref == 'main' + if: "github.base_ref == 'main'" needs: "test" steps: - name: "Create working directory" @@ -70,7 +119,7 @@ jobs: mkdir -p "pulp_python" working-directory: "." - name: "Download Deprecations" - uses: actions/download-artifact@v4 + uses: "actions/download-artifact@v8" with: pattern: "deprecations-*" path: "pulp_python" @@ -84,15 +133,42 @@ jobs: # This is a dummy dependent task to have a single entry for the branch protection rules. runs-on: "ubuntu-latest" needs: + - "check-changes" - "check-commits" - "lint" - "test" - "docs" + - "sanity" if: "always()" steps: - name: "Collect needed jobs results" working-directory: "." run: | - echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result' - echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0' + RUN_TESTS=${{ needs.check-changes.outputs.run_tests }} + RUN_DOCS=${{ needs.check-changes.outputs.run_docs }} + + check_jobs() { + local filter="$1" + local needs_json='${{toJson(needs)}}' + # output failed jobs after filter + echo "$needs_json" | jq -r "to_entries[]|select($filter)|select(.value.result!=\"success\")|.key + \": \" + .value.result" + # fails if not all selected jobs passed + echo "$needs_json" | jq -e "to_entries|map(select($filter))|map(select(.value.result!=\"success\"))|length == 0" + } + + if [ "$RUN_TESTS" == "1" ] && [ "$RUN_DOCS" == "1" ]; then + FILTERS="true" # check all jobs + elif [ "$RUN_TESTS" == "1" ] && [ "$RUN_DOCS" == "0" ]; then + echo "Skipping docs: running on non-default branch" + FILTERS='.key != "docs"' + elif [ "$RUN_TESTS" == "0" ] && [ "$RUN_DOCS" == "1" ]; then + echo "Skipping tests: only doc changes" + FILTERS='.key != "lint" and .key != "test"' + else # RUN_TESTS=0, RUN_DOCS=0 + echo "What is this PR doing??" + FILTERS='.key != "lint" and .key != "test" and .key != "docs"' + fi + + check_jobs "$FILTERS" echo "CI says: Looks good!" +... diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 899147c47..22e5d99b8 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,39 +4,42 @@ # './plugin-template --github pulp_python' to update this file. # # For more info visit https://github.com/pulp/plugin_template +--- name: "Python CodeQL" on: workflow_dispatch: schedule: - - cron: '37 1 * * 6' + - cron: "37 1 * * 6" concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: analyze: - name: Analyze - runs-on: ubuntu-latest + name: "Analyze" + runs-on: "ubuntu-latest" permissions: - actions: read - contents: read - security-events: write + actions: "read" + contents: "read" + security-events: "write" strategy: fail-fast: false matrix: - language: [ 'python' ] + language: + - "python" steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: "Checkout repository" + uses: "actions/checkout@v6" - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + - name: "Initialize CodeQL" + uses: "github/codeql-action/init@v4" with: - languages: ${{ matrix.language }} + languages: "${{ matrix.language }}" - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: "Perform CodeQL Analysis" + uses: "github/codeql-action/analyze@v4" +... diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 9ea307127..6b57c7854 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Create New Release Branch +name: "Create New Release Branch" on: workflow_dispatch: @@ -15,32 +15,33 @@ env: jobs: create-branch: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false + permissions: + contents: "write" + steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install bump-my-version packaging -r plugin_template/requirements.txt - echo ::endgroup:: - name: "Setting secrets" working-directory: "pulp_python" @@ -49,8 +50,8 @@ jobs: env: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: Determine new branch name - working-directory: pulp_python + - name: "Determine new branch name" + working-directory: "pulp_python" run: | # Just to be sure... git checkout main @@ -62,42 +63,43 @@ jobs: fi echo "NEW_BRANCH=${NEW_BRANCH}" >> "$GITHUB_ENV" - - name: Create release branch - working-directory: pulp_python + - name: "Create release branch" + working-directory: "pulp_python" run: | git branch "${NEW_BRANCH}" - - name: Bump version on main branch - working-directory: pulp_python + - name: "Bump version on main branch" + working-directory: "pulp_python" run: | bump-my-version bump --no-commit minor - - name: Remove entries from CHANGES directory - working-directory: pulp_python + - name: "Remove entries from CHANGES directory" + working-directory: "pulp_python" run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + - - name: Update CI branches in template_config - working-directory: plugin_template + - name: "Update CI branches in template_config" + working-directory: "plugin_template" run: | python3 ./plugin-template pulp_python --github --latest-release-branch "${NEW_BRANCH}" git add -A - - name: Make a PR with version bump and without CHANGES/* - uses: peter-evans/create-pull-request@v6 + - name: "Make a PR with version bump and without CHANGES/*" + uses: "peter-evans/create-pull-request@v8" with: - path: pulp_python - token: ${{ secrets.RELEASE_TOKEN }} - committer: pulpbot - author: pulpbot - branch: minor-version-bump - base: main - title: Bump minor version + path: "pulp_python" + token: "${{ secrets.RELEASE_TOKEN }}" + committer: "pulpbot " + author: "pulpbot " + branch: "minor-version-bump" + base: "main" + title: "Bump minor version" commit-message: | Bump minor version delete-branch: true - - name: Push release branch - working-directory: pulp_python + - name: "Push release branch" + working-directory: "pulp_python" run: | git push origin "${NEW_BRANCH}" +... diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 692ba626a..c897fcac2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,49 +6,38 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: "Docs" +name: "Docs CI" on: workflow_call: + inputs: + run_docs: + description: "Whether to run docs jobs" + required: true + type: "string" jobs: - test: - if: "endsWith(github.base_ref, 'main')" - runs-on: "ubuntu-20.04" + changelog: + runs-on: "ubuntu-latest" defaults: run: working-directory: "pulp_python" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: - python-version: "3.11" - - name: "Setup cache key" - run: | - git ls-remote https://github.com/pulp/pulp-docs main | tee pulp-docs-main-sha - - uses: "actions/cache@v4" - with: - path: "~/.cache/pip" - key: ${{ runner.os }}-pip-${{ hashFiles('pulp-docs-main-sha') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: "3.12" - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install -r doc_requirements.txt - echo ::endgroup:: + pip install towncrier - name: "Build changelog" run: | towncrier build --yes --version 4.0.0.ci - - name: "Build docs" - run: | - pulp-docs build - - no-test: - if: "!endsWith(github.base_ref, 'main')" - runs-on: "ubuntu-20.04" - steps: - - run: | - echo "Skip docs testing on non-main branches." + docs: + if: "${{ inputs.run_docs == '1' }}" + uses: "pulp/pulp-docs/.github/workflows/docs-ci.yml@main" + with: + pulpdocs_ref: "main" +... diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9dddcbdd9..d07a39a08 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,51 +19,33 @@ jobs: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install -r lint_requirements.txt - echo ::endgroup:: - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows - - name: "Verify bump version config" - run: | - bump-my-version bump --dry-run release - bump-my-version show-bump - # Lint code. - name: "Run flake8" run: | flake8 - - name: "Run extra lint checks" - run: | - [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - - - name: "Check for any files unintentionally left out of MANIFEST.in" - run: | - check-manifest - - - name: "Verify requirements files" - run: | - python .ci/scripts/check_requirements.py - - - name: "Check for pulpcore imports outside of pulpcore.plugin" - run: | - sh .ci/scripts/check_pulpcore_imports.sh - - name: "Check for common gettext problems" run: | sh .ci/scripts/check_gettext.sh + + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh +... diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f155d81c7..ed8149ce9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ on: schedule: # * is a special character in YAML so you have to quote this string # runs at 3:00 UTC daily - - cron: '00 3 * * *' + - cron: "00 3 * * *" workflow_dispatch: defaults: @@ -34,38 +34,46 @@ jobs: [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] changelog: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: - python-version: "3.11" + python-version: "3.13" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install gitpython packaging toml - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Collect changes from all branches - run: python .ci/scripts/collect_changes.py + - name: "Collect changes from all branches" + run: | + python .ci/scripts/collect_changes.py - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 + - name: "Create Pull Request" + uses: "peter-evans/create-pull-request@v8" + id: "create_pr_changelog" with: - token: ${{ secrets.RELEASE_TOKEN }} + token: "${{ secrets.RELEASE_TOKEN }}" title: "Update Changelog" body: "" branch: "changelog/update" delete-branch: true path: "pulp_python" + - name: "Mark PR automerge" + working-directory: "pulp_python" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_changelog.outputs.pull-request-number }}" + if: "steps.create_pr_changelog.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true ... diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 0e0a7936a..d80548961 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -32,10 +32,10 @@ jobs: permissions: pull-requests: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Determine PR labels" @@ -43,7 +43,7 @@ jobs: pip install GitPython==3.1.42 git fetch origin ${{ github.event.pull_request.head.sha }} python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" - - uses: "actions/github-script@v7" + - uses: "actions/github-script@v8" name: "Apply PR Labels" with: script: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 274490d90..6d0d3aa8c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,50 +23,31 @@ jobs: runs-on: "ubuntu-latest" needs: - "build" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + environment: + name: "pypi" + url: "https://pypi.org/p/pulp-python" + permissions: + id-token: "write" steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_python" - - uses: "actions/download-artifact@v4" with: name: "plugin_package" - path: "pulp_python/dist/" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: + path: "dist/" - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Deploy plugin to pypi" - run: | - .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.ref_name }} + - name: "Publish package to PyPI" + uses: pypa/gh-action-pypi-publish@release/v1 publish-python-bindings: runs-on: "ubuntu-latest" needs: - "build" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + environment: + name: "pypi" + permissions: + id-token: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" @@ -81,35 +62,21 @@ jobs: run: | tar -xvf python-python-client.tar - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: "Publish client to pypi" - run: | - bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.ref_name }} + uses: "pypa/gh-action-pypi-publish@release/v1" + with: + packages-dir: "pulp_python/dist/" publish-ruby-bindings: runs-on: "ubuntu-latest" needs: - "build" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + environment: + name: "rubygems" + permissions: + id-token: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" @@ -128,15 +95,12 @@ jobs: with: ruby-version: "2.6" - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" + - name: "Set RubyGems Credentials" + uses: "rubygems/configure-rubygems-credentials@v1.0.0" - - name: "Publish client to rubygems" + - name: "Publish client to RubyGems" run: | - bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} + gem push "pulp_python_client-${{ github.ref_name }}.gem" create-gh-release: runs-on: "ubuntu-latest" @@ -146,18 +110,50 @@ jobs: - "publish-python-bindings" - "publish-ruby-bindings" + permissions: + contents: "write" + + env: + TAG_NAME: "${{ github.ref_name }}" + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 0 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Install towncrier" + run: | + pip install towncrier + + - name: "Get release notes" + id: "get_release_notes" + shell: "bash" + run: | + # The last commit before the release commit contains the release CHANGES fragments + git checkout "${TAG_NAME}~" + NOTES=$(towncrier build --draft --version $TAG_NAME | .ci/scripts/clean_gh_release_notes.py pulp_python $TAG_NAME) + echo "body<> $GITHUB_OUTPUT + echo "$NOTES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: "Create release on GitHub" - uses: "actions/github-script@v7" + uses: "actions/github-script@v8" env: - TAG_NAME: "${{ github.ref_name }}" + RELEASE_BODY: "${{ steps.get_release_notes.outputs.body }}" with: script: | - const { TAG_NAME } = process.env; + const { TAG_NAME, RELEASE_BODY } = process.env; await github.rest.repos.createRelease({ owner: context.repo.owner, repo: context.repo.repo, tag_name: TAG_NAME, + body: RELEASE_BODY, make_latest: "legacy", }); +... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b758bae60..e056bcfaa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,21 +22,19 @@ jobs: fail-fast: false steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" token: ${{ secrets.RELEASE_TOKEN }} - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install bump-my-version towncrier - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | @@ -58,3 +56,4 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" +... diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml new file mode 100644 index 000000000..223f07cbd --- /dev/null +++ b/.github/workflows/sanity.yml @@ -0,0 +1,54 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + + +# This file describes checks that should prevent a premature merge, +# but still let the tests run for demonstrations or experiments. +--- +name: "Sanity" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + sanity: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + pip install -r lint_requirements.txt + + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest + + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py + + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh +... diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 4cc50fb8f..d4abe77ce 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -7,66 +7,126 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script prepares the scenario definition in the .ci/ansible/vars/main.yaml file. +# +# It requires the following environment: +# TEST - The name of the scenario to prepare. +# +# It may also dump the {lower,upper}bounds_constraints.txt for the specific scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -mveuo pipefail +if [ -f .github/workflows/scripts/pre_before_install.sh ]; then + source .github/workflows/scripts/pre_before_install.sh +fi -if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ] -then - BRANCH_BUILD=0 -else - BRANCH_BUILD=1 - BRANCH="${GITHUB_REF##refs/heads/}" +COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" +COMPONENT_SOURCE="./pulp_python/dist/pulp_python-${COMPONENT_VERSION}-py3-none-any.whl" +if [ "$TEST" = "s3" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" +fi +if [ "$TEST" = "azure" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]" +fi + +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u pyproject.toml > upperbounds_constraints.txt fi -if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ] -then - TAG_BUILD=0 -else - TAG_BUILD=1 - BRANCH="${GITHUB_REF##refs/tags/}" +if [[ "$TEST" = "lowerbounds" ]]; then + python3 .ci/scripts/calc_constraints.py pyproject.toml > lowerbounds_constraints.txt fi -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export COMMIT_MSG +export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") -COMPONENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") +echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" -mkdir .ci/ansible/vars || true -echo "---" > .ci/ansible/vars/main.yaml -echo "legacy_component_name: pulp_python" >> .ci/ansible/vars/main.yaml -echo "component_name: python" >> .ci/ansible/vars/main.yaml -echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml +# Compose the scenario definition. +mkdir -p .ci/ansible/vars -export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh -export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh +cat > .ci/ansible/vars/main.yaml << VARSYAML +--- +scenario: "${TEST}" +plugin_name: "pulp_python" +legacy_component_name: "pulp_python" +component_name: "python" +component_version: "${COMPONENT_VERSION}" +pulp_env: {} +pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} +pulp_scheme: "https" +api_root: "${PULP_API_ROOT}" +image: + name: "pulp" + tag: "ci_build" + ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest" + source: "${COMPONENT_SOURCE}" + ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) + upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) + lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) + webserver_snippet: $(test -f pulp_python/app/webserver_snippets/nginx.conf && echo -n true || echo -n false ) +extra_files: + - origin: "pulp_python" + destination: "pulp_python" +services: + - name: "pulp" + image: "pulp:ci_build" + volumes: + - "./settings:/etc/pulp" + - "../../../pulp-openapi-generator:/root/pulp-openapi-generator" + env: + PULP_WORKERS: "4" + PULP_HTTPS: "true" +VARSYAML -if [ -f $PRE_BEFORE_INSTALL ]; then - source $PRE_BEFORE_INSTALL +if [ "$TEST" = "s3" ]; then + MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA + MINIO_SECRET_KEY=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 + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "minio" + image: "minio/minio" + env: + MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}" + MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}" + command: "server /data" +s3_test: true +minio_access_key: "${MINIO_ACCESS_KEY}" +minio_secret_key: "${MINIO_SECRET_KEY}" +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true} +pulp_scenario_env: {} +VARSYAML fi -if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] -then - echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml +if [ "$TEST" = "azure" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-azurite" + image: "mcr.microsoft.com/azure-storage/azurite" + command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0" +azure_test: true +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "content_origin": null} +pulp_scenario_env: {} +VARSYAML fi -for i in {1..3} -do - ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 -done -if [[ $s -gt 0 ]] -then - echo "Failed to install amazon.aws" - exit $s +if [ "$TEST" = "gcp" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-gcp" + image: "fsouza/fake-gcs-server" + volumes: + - "storage_data:/etc/pulp" + command: " -scheme http" +gcp_test: true +pulp_scenario_settings: null +pulp_scenario_env: {} +VARSYAML fi -if [[ "$TEST" = "pulp" ]]; then - python3 .ci/scripts/calc_constraints.py -u pyproject.toml > upperbounds_constraints.txt -fi -if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_constraints.py pyproject.toml > lowerbounds_constraints.txt -fi +cat >> .ci/ansible/vars/main.yaml << VARSYAML +... +VARSYAML +cat .ci/ansible/vars/main.yaml -if [ -f $POST_BEFORE_INSTALL ]; then - source $POST_BEFORE_INSTALL +if [ -f .github/workflows/scripts/post_before_install.sh ]; then + source .github/workflows/scripts/post_before_install.sh fi diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 5c0072c42..13809a87a 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -7,44 +7,46 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script dumps some files to help understand the setup of the test scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -euv - source .github/workflows/scripts/utils.sh -export PRE_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/pre_before_script.sh -export POST_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/post_before_script.sh - -if [[ -f $PRE_BEFORE_SCRIPT ]]; then - source $PRE_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/pre_before_script.sh ]]; then + source .github/workflows/scripts/pre_before_script.sh fi -# Developers should be able to reproduce the containers with this config -echo "CI vars:" -tail -v -n +1 .ci/ansible/vars/main.yaml - # Developers often want to know the final pulp config -echo "PULP CONFIG:" -tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json +echo +echo "# Pulp config:" +tail -v -n +1 .ci/ansible/settings/settings.* -echo "Containerfile:" +echo +echo "# Containerfile:" tail -v -n +1 .ci/ansible/Containerfile -# Needed for some functional tests -cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" -cmd_prefix bash -c "usermod -a -G wheel pulp" +echo +echo "# Constraints Files:" +# They need not even exist. +tail -v -n +1 ../*/*constraints.txt || true -if [[ "${REDIS_DISABLED:-false}" == true ]]; then - cmd_prefix bash -c "s6-rc -d change redis" - echo "The Redis service was disabled for $TEST" -fi +echo +echo "# pip list outside the container" +pip list + +echo +echo "# pip list inside the container" +cmd_prefix bash -c "pip3 list" + +echo +echo "# State of the containers" +docker ps -a -if [[ -f $POST_BEFORE_SCRIPT ]]; then - source $POST_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/post_before_script.sh ]]; then + source .github/workflows/scripts/post_before_script.sh fi -# Lots of plugins try to use this path, and throw warnings if they cannot access it. -cmd_prefix mkdir /.pytest_cache -cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh index e86ffc293..9eb0def9f 100755 --- a/.github/workflows/scripts/build_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -21,10 +21,10 @@ rm -rf "pulp_python-client" ./gen-client.sh "../pulp_python/python-api.json" "python" python "pulp_python" pushd pulp_python-client -python setup.py sdist bdist_wheel --python-tag py3 +python -m build twine check "dist/pulp_python_client-"*"-py3-none-any.whl" -twine check "dist/pulp_python-client-"*".tar.gz" +twine check "dist/pulp_python_client-"*".tar.gz" tar cvf "../../pulp_python/python-python-client.tar" ./dist diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index a512cbea4..9df47ccde 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -7,130 +7,43 @@ # # For more info visit https://github.com/pulp/plugin_template +set -euv + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. REPO_ROOT="$PWD" -set -euv - source .github/workflows/scripts/utils.sh -PLUGIN_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" -PLUGIN_SOURCE="./pulp_python/dist/pulp_python-${PLUGIN_VERSION}-py3-none-any.whl" - -export PULP_API_ROOT="/pulp/" - PIP_REQUIREMENTS=("pulp-cli") # This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} +if [[ "$TEST" = "s3" ]]; then +for i in {1..3} +do + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 +done +if [[ $s -gt 0 ]] +then + echo "Failed to install amazon.aws" + exit $s +fi +fi + # Check out the pulp-cli branch matching the installed version. PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli cd .ci/ansible/ -if [ "$TEST" = "s3" ]; then - PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[s3]" -fi -if [ "$TEST" = "azure" ]; then - PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[azure]" -fi - -cat >> vars/main.yaml << VARSYAML -image: - name: pulp - tag: "ci_build" -plugins: - - name: pulp_python - source: "${PLUGIN_SOURCE}" -VARSYAML -if [[ -f ../../ci_requirements.txt ]]; then - cat >> vars/main.yaml << VARSYAML - ci_requirements: true -VARSYAML -fi -if [ "$TEST" = "pulp" ]; then - cat >> vars/main.yaml << VARSYAML - upperbounds: true -VARSYAML -fi -if [ "$TEST" = "lowerbounds" ]; then - cat >> vars/main.yaml << VARSYAML - lowerbounds: true -VARSYAML -fi - -cat >> vars/main.yaml << VARSYAML -services: - - name: pulp - image: "pulp:ci_build" - volumes: - - ./settings:/etc/pulp - - ./ssh:/keys/ - - ~/.config:/var/lib/pulp/.config - - ../../../pulp-openapi-generator:/root/pulp-openapi-generator - env: - PULP_WORKERS: "4" - PULP_HTTPS: "true" -VARSYAML - -cat >> vars/main.yaml << VARSYAML -pulp_env: {} -pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} -pulp_scheme: https -pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest -VARSYAML - -if [ "$TEST" = "s3" ]; then - export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA - export MINIO_SECRET_KEY=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 - sed -i -e '/^services:/a \ - - name: minio\ - image: minio/minio\ - env:\ - MINIO_ACCESS_KEY: "'$MINIO_ACCESS_KEY'"\ - MINIO_SECRET_KEY: "'$MINIO_SECRET_KEY'"\ - command: "server /data"' vars/main.yaml - sed -i -e '$a s3_test: true\ -minio_access_key: "'$MINIO_ACCESS_KEY'"\ -minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"domain_enabled": true}\ -pulp_scenario_env: {}\ -test_storages_compat_layer: true\ -' vars/main.yaml - export PULP_API_ROOT="/rerouted/djnd/" -fi - -if [ "$TEST" = "azure" ]; then - sed -i -e '/^services:/a \ - - name: ci-azurite\ - image: mcr.microsoft.com/azure-storage/azurite\ - volumes:\ - - ./azurite:/etc/pulp\ - command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml - sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"domain_enabled": true}\ -pulp_scenario_env: {}\ -' vars/main.yaml -fi - -echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" - -if [ "${PULP_API_ROOT:-}" ]; then - sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml -fi -pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" --username "admin" --password "password" +pulp config create --base-url https://pulp --api-root "${PULP_API_ROOT}" --username "admin" --password "password" cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml ansible-playbook start_container.yaml -# .config needs to be accessible by the pulp user in the container, but some -# files will likely be modified on the host by post/pre scripts. -chmod 777 ~/.config/pulp_smash/ -chmod 666 ~/.config/pulp_smash/settings.json # Plugins often write to ~/.config/pulp/cli.toml from the host chmod 777 ~/.config/pulp chmod 666 ~/.config/pulp/cli.toml @@ -156,6 +69,10 @@ if [[ "$TEST" = "azure" ]]; then az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING fi -echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list" -echo ::endgroup:: +# Needed for some functional tests +cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" +cmd_prefix bash -c "usermod -a -G wheel pulp" + +# Lots of plugins try to use this path, and throw warnings if they cannot access it. +cmd_prefix mkdir /.pytest_cache +cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh deleted file mode 100755 index e771b5e7e..000000000 --- a/.github/workflows/scripts/publish_client_gem.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -VERSION="$1" - -if [[ -z "${VERSION}" ]] -then - echo "No version specified." - exit 1 -fi - -mkdir -p ~/.gem -touch ~/.gem/credentials -echo "--- -:rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials -sudo chmod 600 ~/.gem/credentials -gem push "pulp_python_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh deleted file mode 100755 index 144f2597f..000000000 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -VERSION="$1" - -if [[ -z "${VERSION}" ]] -then - echo "No version specified." - exit 1 -fi - -twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ -"dist/pulp_python_client-${VERSION}-py3-none-any.whl" \ -"dist/pulp_python-client-${VERSION}.tar.gz" \ -; diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh deleted file mode 100755 index 16b0ce055..000000000 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -VERSION="$1" - -if [[ -z "${VERSION}" ]] -then - echo "No version specified." - exit 1 -fi - -twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ -dist/pulp?python-"${VERSION}"-py3-none-any.whl \ -dist/pulp?python-"${VERSION}".tar.gz \ -; diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 8e60f87d1..5a4910e30 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -28,6 +28,8 @@ export PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" REPORTED_STATUS="$(pulp status)" +echo "${REPORTED_STATUS}" + echo "machine pulp login admin password password @@ -70,7 +72,7 @@ pushd ../pulp-openapi-generator rm -rf "./${PACKAGE}-client" ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" pushd "${PACKAGE}-client" - python setup.py sdist bdist_wheel --python-tag py3 + python -m build popd else if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] @@ -98,8 +100,11 @@ echo "::endgroup::" # Install test requirements ########################### +# Carry on previous constraints (there might be no such file). +cat *_constraints.txt > bindings_constraints.txt || true +cat .ci/assets/ci_constraints.txt >> bindings_constraints.txt # Add a safeguard to make sure the proper versions of the clients are installed. -echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' >> bindings_constraints.txt cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt @@ -114,7 +119,7 @@ echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations python --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --pyargs pulp_python.tests.unit" +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulp_python.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then @@ -130,16 +135,16 @@ if [ -f "$FUNC_TEST_SCRIPT" ]; then else if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] then - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" else - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel'" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel'" fi fi pushd ../pulp-cli pip install -r test_requirements.txt -pytest -v -m "pulp_python" +pytest -v tests -m "pulp_python" popd if [ -f "$POST_SCRIPT" ]; then diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a18b9230a..fbddb788b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ on: inputs: matrix_env: required: true - type: string + type: "string" defaults: run: @@ -24,38 +24,38 @@ jobs: strategy: fail-fast: false matrix: - env: ${{ fromJSON(inputs.matrix_env) }} + env: "${{ fromJSON(inputs.matrix_env) }}" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Download plugin package" - uses: "actions/download-artifact@v4" + uses: "actions/download-artifact@v8" with: name: "plugin_package" path: "pulp_python/dist/" - name: "Download API specs" - uses: "actions/download-artifact@v4" + uses: "actions/download-artifact@v8" with: name: "api_spec" path: "pulp_python/" - name: "Download client packages" - uses: "actions/download-artifact@v4" + uses: "actions/download-artifact@v8" with: name: "python-client.tar" path: "pulp_python" @@ -70,16 +70,14 @@ jobs: - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version + pip install build towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - name: "Set environment variables" run: | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - name: "Before Install" + - name: "Prepare Scenario Definition" run: | .github/workflows/scripts/before_install.sh shell: "bash" @@ -99,7 +97,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - name: "Before Script" + - name: "Dump CI Metadata" run: | .github/workflows/scripts/before_script.sh shell: "bash" @@ -108,7 +106,6 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" - name: "Script" run: | @@ -125,7 +122,7 @@ jobs: docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt - name: "Upload Deprecations" - uses: actions/upload-artifact@v4 + uses: "actions/upload-artifact@v5" with: name: "deprecations-${{ matrix.env.TEST }}" path: "pulp_python/deprecations-${{ matrix.env.TEST }}.txt" diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index 4565da8cd..39320dcae 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -19,7 +19,7 @@ jobs: update_backport_labels: runs-on: "ubuntu-latest" steps: - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Configure Git with pulpbot name and email" @@ -28,12 +28,11 @@ jobs: git config --global user.email 'pulp-infra@redhat.com' - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install requests pyyaml - echo ::endgroup:: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" - name: "Update labels" run: | python3 .github/workflows/scripts/update_backport_labels.py env: GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" +... diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 46ceadfd4..89677b6e8 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -12,7 +12,7 @@ on: schedule: # * is a special character in YAML so you have to quote this string # runs at 2:30 UTC every Sunday - - cron: '30 2 * * 0' + - cron: "30 2 * * 0" workflow_dispatch: jobs: @@ -23,27 +23,25 @@ jobs: fail-fast: false steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install gitpython packaging -r plugin_template/requirements.txt - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -55,7 +53,7 @@ jobs: ../plugin_template/scripts/update_ci.sh - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_main" with: token: "${{ secrets.RELEASE_TOKEN }}" @@ -74,7 +72,7 @@ jobs: env: GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" continue-on-error: true - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -86,7 +84,7 @@ jobs: ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_3_11" with: token: "${{ secrets.RELEASE_TOKEN }}" @@ -105,7 +103,7 @@ jobs: env: GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" continue-on-error: true - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -117,7 +115,7 @@ jobs: ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_3_12" with: token: "${{ secrets.RELEASE_TOKEN }}" diff --git a/CHANGES.md b/CHANGES.md index df585515b..7c28be24c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,72 @@ [//]: # (towncrier release notes start) +## 3.13.5 (2025-04-23) {: #3.13.5 } + +No significant changes. + +--- + +## 3.13.4 (2025-04-10) {: #3.13.4 } + +#### Misc {: #3.13.4-misc } + +- [#809](https://github.com/pulp/pulp_python/issues/809) + +--- + +## 3.13.3 (2025-04-07) {: #3.13.3 } + +#### Bugfixes {: #3.13.3-bugfix } + +- Fixed a proxy sync regression introduced in 3.13.0. + +--- + +## 3.13.2 (2025-02-20) {: #3.13.2 } + +#### Misc {: #3.13.2-misc } + +- + +--- + +## 3.13.1 (2025-02-11) {: #3.13.1 } + +No significant changes. + +--- + +## 3.13.0 (2025-02-05) {: #3.13.0 } + +#### Features {: #3.13.0-feature } + +- Added pulpcore 3.70 compatibility + +#### Bugfixes {: #3.13.0-bugfix } + +- Fixed uploads not supporting packages using metadata spec 2.3 + [#682](https://github.com/pulp/pulp_python/issues/682) +- Fixed the `package_types` filter breaking other remote filters. + [#691](https://github.com/pulp/pulp_python/issues/691) +- Fixed package name normalization issue preventing syncing packages with "." or "_" in their names. + [#716](https://github.com/pulp/pulp_python/issues/716) +- Fixed replicate failing on upstream on-demand repositories + [#718](https://github.com/pulp/pulp_python/issues/718) +- Fixed `requires_python` field not being properly set on package upload. + + Run the new `pulpcore-manager repair-python-metadata` command with repositories containing affected + packages to repair their metadata. + [#773](https://github.com/pulp/pulp_python/issues/773) +- Fixed the JSONField specification so it doesn't break ruby bindings. + See context [here](https://github.com/pulp/pulp_rpm/issues/3639). + +#### Misc {: #3.13.0-misc } + +- [#774](https://github.com/pulp/pulp_python/issues/774) + +--- + ## 3.12.5 (2024-10-25) {: #3.12.5 } #### Bugfixes {: #3.12.5-bugfix } diff --git a/CHANGES/+fix-any-type.bugfix b/CHANGES/+fix-any-type.bugfix deleted file mode 100644 index 89ea25f51..000000000 --- a/CHANGES/+fix-any-type.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed the JSONField specification so it doesn't break ruby bindings. -See context [here](https://github.com/pulp/pulp_rpm/issues/3639). diff --git a/CHANGES/+pulpcore-3.70.feature b/CHANGES/+pulpcore-3.70.feature deleted file mode 100644 index b70054c1f..000000000 --- a/CHANGES/+pulpcore-3.70.feature +++ /dev/null @@ -1 +0,0 @@ -Added pulpcore 3.70 compatibility diff --git a/CHANGES/682.bugfix b/CHANGES/682.bugfix deleted file mode 100644 index 4bf3e5b52..000000000 --- a/CHANGES/682.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed uploads not supporting packages using metadata spec 2.3 diff --git a/CHANGES/691.bugfix b/CHANGES/691.bugfix deleted file mode 100644 index 448af7291..000000000 --- a/CHANGES/691.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed the `package_types` filter breaking other remote filters. diff --git a/CHANGES/716.bugfix b/CHANGES/716.bugfix deleted file mode 100644 index 344bbf230..000000000 --- a/CHANGES/716.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed package name normalization issue preventing syncing packages with "." or "_" in their names. diff --git a/CHANGES/718.bugfix b/CHANGES/718.bugfix deleted file mode 100644 index 6f546fee7..000000000 --- a/CHANGES/718.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed replicate failing on upstream on-demand repositories diff --git a/CHANGES/773.bugfix b/CHANGES/773.bugfix deleted file mode 100644 index cc947f699..000000000 --- a/CHANGES/773.bugfix +++ /dev/null @@ -1,4 +0,0 @@ -Fixed `requires_python` field not being properly set on package upload. - -Run the new `pulpcore-manager repair-python-metadata` command with repositories containing affected -packages to repair their metadata. diff --git a/CHANGES/774.misc b/CHANGES/774.misc deleted file mode 100644 index 274896932..000000000 --- a/CHANGES/774.misc +++ /dev/null @@ -1 +0,0 @@ -Rebase migrations to prepare for pulpcore 3.70. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 1a4d476e8..9bf86307f 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -10,7 +10,7 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.13.0.dev" + version = "3.13.6.dev" python_package_name = "pulp-python" domain_compatible = True diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index 60b3c04d5..b6210e2b4 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -51,7 +51,8 @@ log = logging.getLogger(__name__) -BASE_CONTENT_URL = urljoin(settings.CONTENT_ORIGIN, settings.CONTENT_PATH_PREFIX) +ORIGIN_HOST = settings.CONTENT_ORIGIN if settings.CONTENT_ORIGIN else settings.PYPI_API_HOSTNAME +BASE_CONTENT_URL = urljoin(ORIGIN_HOST, settings.CONTENT_PATH_PREFIX) class PyPIMixin: diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 213c4a7df..317f7b6bc 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -3,9 +3,11 @@ from aiohttp import ClientResponseError, ClientError from lxml.etree import LxmlError from gettext import gettext as _ +from functools import partial from rest_framework import serializers +from pulpcore.plugin.download import HttpDownloader from pulpcore.plugin.models import Artifact, ProgressReport, Remote, Repository from pulpcore.plugin.stages import ( DeclarativeArtifact, @@ -112,11 +114,22 @@ async def run(self): """ # Bandersnatch includes leading slash when forming API urls url = self.remote.url.rstrip("/") + downloader = self.remote.get_downloader(url=url) + if not isinstance(downloader, HttpDownloader): + raise ValueError("Only HTTP(S) is supported for python syncing") + async with Master(url) as master: # Replace the session with the remote's downloader session old_session = master.session - factory = self.remote.download_factory - master.session = factory._session + master.session = downloader.session + + # Set up master.get with remote's auth & proxy settings + master.get = partial( + master.get, + auth=downloader.auth, + proxy=downloader.proxy, + proxy_auth=downloader.proxy_auth, + ) deferred_download = self.remote.policy != Remote.IMMEDIATE workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 19007ab61..c3d350fa4 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -315,7 +315,8 @@ def find_artifact(): content_artifact = content.contentartifact_set.first() artifact = find_artifact() - origin = settings.CONTENT_ORIGIN.strip("/") + origin = settings.CONTENT_ORIGIN or settings.PYPI_API_HOSTNAME or "" + origin = origin.strip("/") prefix = settings.CONTENT_PATH_PREFIX.strip("/") base_path = base_path.strip("/") components = [origin, prefix, base_path, content.filename] diff --git a/pulp_python/tests/functional/api/test_consume_content.py b/pulp_python/tests/functional/api/test_consume_content.py index 9c153cd10..98ac094ee 100644 --- a/pulp_python/tests/functional/api/test_consume_content.py +++ b/pulp_python/tests/functional/api/test_consume_content.py @@ -21,6 +21,7 @@ def test_pip_consume_content( "install", "--no-deps", "--no-cache-dir", + "--no-build-isolation", "--force-reinstall", "--trusted-host", urlsplit(distro.base_url).hostname, diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index 6dcbc4505..4264d0512 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -134,3 +134,16 @@ def test_upload_requires_python(python_content_factory): content = python_content_factory(filename, url=package.url) assert content.requires_python == ">=3.8" break + + +@pytest.mark.parallel +def test_upload_metadata_24_spec(python_content_factory): + """Test that packages using metadata spec 2.4 can be uploaded to pulp.""" + filename = "urllib3-2.3.0-py3-none-any.whl" + with PyPISimple() as client: + page = client.get_project_page("urllib3") + for package in page.packages: + if package.filename == filename: + content = python_content_factory(filename, url=package.url) + assert content.metadata_version == "2.4" + break diff --git a/pulp_python/tests/functional/api/test_domains.py b/pulp_python/tests/functional/api/test_domains.py index 786e58ecb..0f5233a44 100644 --- a/pulp_python/tests/functional/api/test_domains.py +++ b/pulp_python/tests/functional/api/test_domains.py @@ -268,6 +268,7 @@ def test_domain_pypi_apis( "pip", "install", "--no-deps", + "--no-build-isolation", "--trusted-host", urlsplit(distro.base_url).hostname, "-i", diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 1e557f367..034f8f83f 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -224,26 +224,6 @@ def test_twine_upload( check=True, ) - # Test re-uploading same packages with --skip-existing works - output = subprocess.run( - ( - "twine", - "upload", - "--repository-url", - url, - dist_dir / "*", - "-u", - username, - "-p", - password, - "--skip-existing", - ), - capture_output=True, - check=True, - text=True - ) - assert output.stdout.count("Skipping") == 2 - @pytest.mark.parallel def test_simple_redirect_with_publications( diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index d371db250..231ae1bcf 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -124,18 +124,22 @@ "aiohttp", # matches 7 "bcrypt", # matches 8 "celery", # matches 13 + "crane", # matches 0 "Django", # matches 31 + "pulp-2to3-migration", # matches 2 "pytz", # matches 6 "scipy", # matches 23 + "setuptools", # matches 2 "shelf-reader", # matches 2 + "twine", # matches 14 ] -PYTHON_LG_PACKAGE_COUNT = 90 +PYTHON_LG_PACKAGE_COUNT = 108 PYTHON_LG_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_LG_PACKAGE_COUNT} PYTHON_LG_FIXTURE_COUNTS = { - "latest_3": 49, - "sdist": 27, - "bdist_wheel": 63, - "multi": 33, # keep_latest=1, package_types="bdist_wheel", prereleases=False + "latest_3": 59, + "sdist": 36, + "bdist_wheel": 72, + "multi": 36, # keep_latest=1, package_types="bdist_wheel", prereleases=False } DJANGO_LATEST_3 = 4 # latest version has 2 dists, each other has 1 diff --git a/pyproject.toml b/pyproject.toml index 9cc858dc6..7797a92e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = 'setuptools.build_meta' [project] name = "pulp-python" -version = "3.13.0.dev" +version = "3.13.6.dev" description = "pulp-python plugin for the Pulp Project" readme = "README.md" authors = [ @@ -28,8 +28,8 @@ classifiers=[ requires-python = ">=3.9" dependencies = [ "pulpcore>=3.49.0,<3.85", - "pkginfo>=1.10.0,<1.12.0", # Twine has <1.11 in their requirements - "bandersnatch>=6.3,<7.0", # Anything >6.3 requires Python 3.10+ + "pkginfo>=1.10.0,<1.13.0", + "bandersnatch>=6.3.0,<6.4", # Anything >6.3 requires Python 3.10+ "pypi-simple>=1.5.0,<2.0", ] @@ -72,12 +72,14 @@ ignore = [ ".github/**", "lint_requirements.txt", ".flake8", + "AGENTS.md", + "CLAUDE.md", ] [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.13.0.dev" +current_version = "3.13.6.dev" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" @@ -121,4 +123,4 @@ replace = "version = \"{new_version}\"" filename = "./pyproject.toml" search = "version = \"{current_version}\"" -replace = "version = \"{new_version}\"" \ No newline at end of file +replace = "version = \"{new_version}\"" diff --git a/template_config.yml b/template_config.yml index d84bdba04..05480d07a 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,64 +1,95 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-420-gf332a34 +# generated with plugin_template +# +# After editing this file please always reapply the plugin template before committing any changes. -api_root: /pulp/ +--- +api_root: "/pulp/" black: false check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true -ci_base_image: ghcr.io/pulp/pulp-ci-centos9 +ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" ci_env: {} -ci_trigger: '{pull_request: {branches: [''*'']}}' -cli_package: pulp-cli -cli_repo: https://github.com/pulp/pulp-cli.git +ci_trigger: "{pull_request: {branches: ['*']}}" +cli_package: "pulp-cli" +cli_repo: "https://github.com/pulp/pulp-cli.git" core_import_allowed: [] deploy_client_to_pypi: true deploy_client_to_rubygems: true deploy_to_pypi: true disabled_redis_runners: [] docker_fixtures: false +extra_files: [] flake8: true flake8_ignore: [] -github_org: pulp -latest_release_branch: '3.12' +github_org: "pulp" +latest_release_branch: "3.12" lint_requirements: true os_required_packages: [] parallel_test_workers: 8 -plugin_app_label: python -plugin_default_branch: main -plugin_name: pulp_python +plugin_app_label: "python" +plugin_default_branch: "main" +plugin_name: "pulp_python" plugins: -- app_label: python - name: pulp_python -post_job_template: null -pre_job_template: null + - app_label: "python" + name: "pulp_python" pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} pulp_env_s3: {} -pulp_scheme: https +pulp_scheme: "https" pulp_settings: - allowed_export_paths: /tmp - allowed_import_paths: /tmp + allowed_export_paths: "/tmp" + allowed_import_paths: "/tmp" orphan_protection_time: 0 - pypi_api_hostname: https://pulp:443 + pypi_api_hostname: "https://pulp:443" pulp_settings_azure: - domain_enabled: true + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.azure_storage.AzureStorage" + OPTIONS: + account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + account_name: "devstoreaccount1" + azure_container: "pulp-test" + connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;" + expiration_secs: 120 + location: "pulp3" + overwrite_files: true + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" + content_origin: null pulp_settings_gcp: null pulp_settings_s3: + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.s3boto3.S3Boto3Storage" + OPTIONS: + access_key: "AKIAIT2Z5TDYPX3ARJBA" + addressing_style: "path" + bucket_name: "pulp3" + default_acl: "@none" + endpoint_url: "http://minio:9000" + region_name: "eu-central-1" + secret_key: "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS" + signature_version: "s3v4" + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" domain_enabled: true pydocstyle: true -release_email: pulp-infra@redhat.com -release_user: pulpbot +release_email: "pulp-infra@redhat.com" +release_user: "pulpbot" stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 stalebot_limit_to_pulls: true supported_release_branches: -- '3.11' + - "3.11" sync_ci: true test_azure: true test_cli: true @@ -66,8 +97,7 @@ test_deprecations: true test_gcp: false test_lowerbounds: true test_performance: false -test_reroute: true test_s3: true -test_storages_compat_layer: true use_issue_template: true +...