diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index a6d10ab4f..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[bumpversion] -current_version = 3.12.0.dev -commit = False -tag = False -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? -serialize = - {major}.{minor}.{patch}.{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = prod -first_value = dev -values = - dev - prod - -[bumpversion:file:./pulp_python/app/__init__.py] - -[bumpversion:file:./setup.py] - -[bumpversion:file:./docs/conf.py] diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index bad75f3bb..fb9ed4a1c 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,28 +1,29 @@ -FROM {{ ci_base | default(pulp_default_container) }} +FROM {{ image.ci_base }} +{%- if image.webserver_snippet %} -# Add source directories to container -{% for item in plugins %} -ADD ./{{ item.name }} ./{{ item.name }} -{% endfor %} +ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf +{%- endif %} + +{%- for item in extra_files | default([]) %} -# Install python packages -# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads -# Hacking botocore (https://github.com/boto/botocore/pull/1990) +ADD ./{{ item.origin }} {{ item.destination }} +{%- endfor %} -RUN pip3 install -{%- if s3_test | default(false) -%} -{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue +# This MUST be the ONLY call to pip install in inside the container. +RUN pip3 install --upgrade pip setuptools wheel && \ + rm -rf /root/.cache/pip && \ + pip3 install {{ image.source }} +{%- if image.upperbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt {%- endif -%} -{%- for item in plugins -%} -{{ " " }}{{ item.source }} -{%- if item.lowerbounds | default(false) -%} -{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt +{%- if image.lowerbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt {%- endif -%} -{%- if item.ci_requirements | default(false) -%} -{{ " " }}-r ./{{ item.name }}/ci_requirements.txt +{%- if image.ci_requirements | default(false) -%} +{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt {%- endif -%} -{%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt +{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \ + rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} @@ -39,11 +40,8 @@ ENV {{ key | upper }}={{ value }} USER pulp:pulp RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link -USER root:root -{% for item in plugins %} -RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ - ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true -{% endfor %} +RUN mkdir /var/lib/pulp/.config +USER root:root ENTRYPOINT ["/init"] diff --git a/.ci/ansible/build_container.yaml b/.ci/ansible/build_container.yaml index c380b430a..0a188cba9 100644 --- a/.ci/ansible/build_container.yaml +++ b/.ci/ansible/build_container.yaml @@ -1,15 +1,14 @@ # Ansible playbook to create the pulp service containers image --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Generate Containerfile from template" - template: - src: Containerfile.j2 - dest: Containerfile - + ansible.builtin.template: + src: "Containerfile.j2" + dest: "Containerfile" - name: "Build pulp image" # We build from the ../.. (parent dir of pulpcore git repo) Docker build # "context" so that repos like pulp-smash are accessible to Docker diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 4ed28f42a..dfe2851de 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -26,35 +26,3 @@ API_ROOT = {{ api_root | repr }} {% endfor %} {% endif %} -{% if s3_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -MEDIA_ROOT = "" -AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" -AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" -AWS_S3_REGION_NAME = "eu-central-1" -AWS_S3_ADDRESSING_STYLE = "path" -S3_USE_SIGV4 = True -AWS_S3_SIGNATURE_VERSION = "s3v4" -AWS_STORAGE_BUCKET_NAME = "pulp3" -AWS_S3_ENDPOINT_URL = "http://minio:9000" -AWS_DEFAULT_ACL = "@none None" -{% endif %} - -{% if azure_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" -MEDIA_ROOT = "" -AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -AZURE_ACCOUNT_NAME = "devstoreaccount1" -AZURE_CONTAINER = "pulp-test" -AZURE_LOCATION = "pulp3" -AZURE_OVERWRITE_FILES = True -AZURE_URL_EXPIRATION_SECS = 120 -AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' -{% endif %} - -{% if gcp_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" -MEDIA_ROOT = "" -GS_BUCKET_NAME = "gcppulp" -GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443" -{% endif %} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 47e5221e5..acdc22ad3 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -1,33 +1,26 @@ # Ansible playbook to start the pulp service container and its supporting services --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Create Settings Directories" - file: + ansible.builtin.file: path: "{{ item }}" - state: directory + state: "directory" mode: "0755" loop: - - settings - - ssh - - ~/.config/pulp_smash + - "settings" - name: "Generate Pulp Settings" template: - src: settings.py.j2 - dest: settings/settings.py - - - name: "Configure pulp-smash" - copy: - src: smash-config.json - dest: ~/.config/pulp_smash/settings.json + src: "settings.py.j2" + dest: "settings/settings.py" - name: "Setup docker networking" docker_network: - name: pulp_ci_bridge + name: "pulp_ci_bridge" - name: "Start Service Containers" docker_container: @@ -37,24 +30,24 @@ recreate: true privileged: true networks: - - name: pulp_ci_bridge + - name: "pulp_ci_bridge" aliases: "{{ item.name }}" volumes: "{{ item.volumes | default(omit) }}" env: "{{ item.env | default(omit) }}" command: "{{ item.command | default(omit) }}" - state: started + state: "started" loop: "{{ services | default([]) }}" - name: "Retrieve Docker Network Info" docker_network_info: - name: pulp_ci_bridge - register: pulp_ci_bridge_info + name: "pulp_ci_bridge" + register: "pulp_ci_bridge_info" - name: "Update /etc/hosts" lineinfile: - path: /etc/hosts + path: "/etc/hosts" regexp: "\\s{{ item.value.Name }}\\s*$" - line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}" + line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}" loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}" become: true @@ -63,19 +56,19 @@ aws_access_key: "{{ minio_access_key }}" aws_secret_key: "{{ minio_secret_key }}" s3_url: "http://minio:9000" - region: eu-central-1 - name: pulp3 - state: present - when: s3_test | default(false) + region: "eu-central-1" + name: "pulp3" + state: "present" + when: "s3_test | default(false)" - block: - name: "Wait for Pulp" uri: url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/" - follow_redirects: all - validate_certs: no - register: result - until: result.status == 200 + follow_redirects: "all" + validate_certs: "no" + register: "result" + until: "result.status == 200" retries: 12 delay: 5 rescue: @@ -86,7 +79,7 @@ - name: "Check version of component being tested" assert: that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver) + - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)" fail_msg: | Component {{ item.app_label }} was expected to be installed in version {{ component_version }}. Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}. @@ -100,9 +93,20 @@ login admin password password -- hosts: pulp +- hosts: "pulp" gather_facts: false tasks: + - name: "Create directory for pulp-smash config" + ansible.builtin.file: + path: "/var/lib/pulp/.config/pulp_smash/" + state: "directory" + mode: "0755" + + - name: "Configure pulp-smash" + ansible.builtin.copy: + src: "smash-config.json" + dest: "/var/lib/pulp/.config/pulp_smash/settings.json" + - name: "Set pulp admin password" command: cmd: "pulpcore-manager reset-admin-password --password password" diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index 2617a4089..c9198f19a 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -1,7 +1,19 @@ # Pulpcore versions without the openapi command do no longer work in the CI -pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* - +# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70 +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.* tablib!=3.6.0 # 3.6.0: This release introduced a regression removing the "html" optional dependency. + + +multidict!=6.3.0 +# This release failed the lower bounds test for some case sensitivity in CIMultiDict. + + +azure-storage-blob!=12.28.* +# Apparently does not work with current azurite. + + +pycares<5 +# older aiodns versions don't pin pycares UB, and are broken by pycares>=5 diff --git a/.ci/assets/release_requirements.txt b/.ci/assets/release_requirements.txt index c064e9477..6635a8723 100644 --- a/.ci/assets/release_requirements.txt +++ b/.ci/assets/release_requirements.txt @@ -1,3 +1,3 @@ -bump2version +bump-my-version gitpython towncrier diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py new file mode 100755 index 000000000..66c494e97 --- /dev/null +++ b/.ci/scripts/calc_constraints.py @@ -0,0 +1,142 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import fileinput +import urllib.request +import sys +from packaging.requirements import Requirement +from packaging.version import Version +import yaml + +try: + import tomllib +except ImportError: + import tomli as tomllib + + +CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" + + +def fetch_pulpcore_upper_bound(requirement): + with urllib.request.urlopen(CORE_TEMPLATE_URL) as f: + template = yaml.safe_load(f.read()) + supported_versions = template["supported_release_branches"] + supported_versions.append(template["latest_release_branch"]) + applicable_versions = sorted( + requirement.specifier.filter((Version(v) for v in supported_versions)) + ) + if len(applicable_versions) == 0: + raise Exception("No supported pulpcore version in required range.") + return f"{requirement.name}~={applicable_versions[-1]}" + + +def split_comment(line): + split_line = line.split("#", maxsplit=1) + try: + comment = " # " + split_line[1].strip() + except IndexError: + comment = "" + return split_line[0].strip(), comment + + +def to_upper_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + if requirement.name == "pulpcore": + # An exception to allow for pulpcore deprecation policy. + return fetch_pulpcore_upper_bound(requirement) + # skip requirement with environment scopes. E.g 'foo==1.0.0;python_version>=3.9' + if requirement.marker: + return f"# ENVIRONMENT IS UNTRACKABLE: {req}" + for spec in requirement.specifier: + if spec.operator == "~=": + return f"# NO BETTER CONSTRAINT: {req}" + if spec.operator == "<=": + operator = "==" + max_version = spec.version + return f"{requirement.name}{operator}{max_version}" + if spec.operator == "<": + operator = "~=" + version = Version(spec.version) + if version.micro != 0: + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" + elif version.minor != 0: + max_version = f"{version.major}.{version.minor - 1}" + elif version.major != 0: + max_version = f"{version.major - 1}.0" + else: + return f"# NO BETTER CONSTRAINT: {req}" + return f"{requirement.name}{operator}{max_version}" + return f"# NO UPPER BOUND: {req}" + + +def to_lower_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + for spec in requirement.specifier: + if spec.operator == ">=": + min_version = spec.version + if requirement.name == "pulpcore": + # Currently an exception to allow for pulpcore bugfix releases. + # TODO Semver libraries should be allowed too. + operator = "~=" + if len(Version(min_version).release) != 3: + raise RuntimeError("Pulpcore lower bound must be in the form '>=x.y.z'.") + else: + operator = "==" + return f"{requirement.name}{operator}{min_version}" + return f"# NO LOWER BOUND: {req}" + + +def main(): + """Calculate constraints for the lower bound of dependencies where possible.""" + parser = argparse.ArgumentParser( + prog=sys.argv[0], + description="Calculate constraints for the lower or upper bound of dependencies where " + "possible.", + ) + parser.add_argument("-u", "--upper", action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true") + parser.add_argument("filename", nargs="*") + args = parser.parse_args() + + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) + else: + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/calc_deps_lowerbounds.py b/.ci/scripts/calc_deps_lowerbounds.py deleted file mode 100755 index 2bb48e250..000000000 --- a/.ci/scripts/calc_deps_lowerbounds.py +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -from packaging.requirements import Requirement - - -def main(): - """Calculate the lower bound of dependencies where possible.""" - with open("requirements.txt") as req_file: - for line in req_file: - try: - requirement = Requirement(line) - except ValueError: - print(line.strip()) - else: - for spec in requirement.specifier: - if spec.operator == ">=": - if requirement.name == "pulpcore": - operator = "~=" - else: - operator = "==" - min_version = str(spec)[2:] - print(f"{requirement.name}{operator}{min_version}") - break - else: - print(line.strip()) - - -if __name__ == "__main__": - main() diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 4f2f3172a..611c882ca 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -1,28 +1,31 @@ #!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# "pyyaml>=6.0.3,<6.1.0", +# ] +# /// import argparse import re import os +import sys +import tomllib +import typing as t +from pathlib import Path + import yaml -from tempfile import TemporaryDirectory from packaging.version import Version from git import Repo -UPSTREAM_REMOTE = "https://github.com/pulp/pulp_python.git" -DEFAULT_BRANCH = "main" RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" -Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"] -Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"] +Y_CHANGELOG_EXTS = [".feature"] +Z_CHANGELOG_EXTS = [".bugfix", ".misc"] -def main(): +def options() -> argparse.Namespace: """Check which branches need a release.""" parser = argparse.ArgumentParser() parser.add_argument( @@ -32,90 +35,157 @@ def main(): "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) - opts = parser.parse_args() - - with TemporaryDirectory() as d: - # Clone from upstream to ensure we have updated branches & main - repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none") - heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")] - available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] - available_branches.sort(key=lambda ver: Version(ver)) - available_branches.append(DEFAULT_BRANCH) - - branches = opts.branches - if branches == "supported": - with open(f"{d}/template_config.yml", mode="r") as f: - tc = yaml.safe_load(f) - branches = set(tc["supported_release_branches"]) - latest_release_branch = tc["latest_release_branch"] - if latest_release_branch is not None: - branches.add(latest_release_branch) - branches.add(DEFAULT_BRANCH) + parser.add_argument( + "--no-fetch", + default=False, + action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true", + help="Don't fetch remote. Run faster at the expense of maybe being outdated.", + ) + return parser.parse_args() + + +def template_config() -> dict[str, t.Any]: + # Assume this script lies in .ci/scripts + path = Path(__file__).absolute().parent.parent.parent / "template_config.yml" + return yaml.safe_load(path.read_text()) + + +def current_version(repo: Repo, commitish: str) -> Version: + try: + pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml")) + try: + current_version = pyproject_toml["project"]["version"] + except Exception: + current_version = pyproject_toml["tool"]["bumpversion"]["current_version"] + except Exception: + current_version = repo.git.grep( + "current_version", commitish, "--", ".bumpversion.cfg" + ).split("=")[-1] + return Version(current_version) + + +def check_pyproject_dependencies(repo: Repo, from_commit: str, to_commit: str) -> list[str]: + try: + new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml")) + try: + new_dependencies = set(new_pyproject["project"]["dependencies"]) + except KeyError: + # New branch does not declare dependencies in pyproject.toml. + # Assume no release needed for this reason. + return [] + old_pyproject = tomllib.loads(repo.git.show(f"{from_commit}:pyproject.toml")) + old_dependencies = set(old_pyproject["project"]["dependencies"]) + if old_dependencies != new_dependencies: + return ["dependencies"] else: - branches = set(branches.split(",")) - - if diff := branches - set(available_branches): - print(f"Supplied branches contains non-existent branches! {diff}") - exit(1) - - print(f"Checking for releases on branches: {branches}") - - releases = [] - for branch in branches: - if branch != DEFAULT_BRANCH: - # Check if a Z release is needed - changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") - z_changelog = False - for change in changes.split("\n"): - # Check each changelog file to make sure everything checks out - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - print( - f"Warning: A non-backported changelog ({change}) is present in the " - f"{branch} release branch!" - ) - elif ext in Z_CHANGELOG_EXTS: - z_changelog = True - - last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}") - req_txt_diff = repo.git.diff( - f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" - ) - if z_changelog or req_txt_diff: - curr_version = Version(last_tag) - assert curr_version.base_version.startswith( - branch - ), "Current-version has to belong to the current branch!" - next_version = Version(f"{branch}.{curr_version.micro + 1}") - reason = "CHANGES" if z_changelog else "requirements.txt" + return [] + except Exception as e: + print(f"WARNING: Comparing the dependencies in pyproject.toml failed. ({e})") + # Gathering more details failed. + return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."] + + +def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int: + DEFAULT_BRANCH: str = template_config["plugin_default_branch"] + + repo = Repo() + + upstream_default_branch = next( + (branch for branch in repo.branches if branch.name == DEFAULT_BRANCH) + ).tracking_branch() + remote = upstream_default_branch.remote_name + if not options.no_fetch: + repo.remote(remote).fetch() + + # Warning: This will not work if branch names contain "/" but we don't really care here. + heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")] + available_branches = [h for h in heads if re.fullmatch(RELEASE_BRANCH_REGEX, h)] + available_branches.sort(key=lambda ver: Version(ver)) + available_branches.append(DEFAULT_BRANCH) + + branches = options.branches + if branches == "supported": + tc = yaml.safe_load(repo.git.show(f"{upstream_default_branch}:template_config.yml")) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) + else: + branches = set(branches.split(",")) + + if diff := branches - set(available_branches): + print(f"Supplied branches contains non-existent branches! {diff}") + return 1 + + branches = [branch for branch in available_branches if branch in branches] + branches.reverse() + + print(f"Checking for releases on branches: {branches}") + + releases = [] + for branch in branches: + if branch != DEFAULT_BRANCH: + # Check if a Z release is needed + reasons = [] + changes = repo.git.ls_tree("-r", "--name-only", f"{remote}/{branch}", "CHANGES/") + z_changelog = False + for change in changes.split("\n"): + # Check each changelog file to make sure everything checks out + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: print( - f"A Z-release is needed for {branch}, " - f"Prev: {last_tag}, " - f"Next: {next_version.base_version}, " - f"Reason: {reason}" + f"Warning: A non-backported changelog ({change}) is present in the " + f"{branch} release branch!" ) + elif ext in Z_CHANGELOG_EXTS: + z_changelog = True + if z_changelog: + reasons.append("Backports") + + last_tag = repo.git.describe("--tags", "--abbrev=0", f"{remote}/{branch}") + req_txt_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "requirements.txt" + ) + if req_txt_diff: + reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.extend(check_pyproject_dependencies(repo, last_tag, f"{remote}/{branch}")) + + if reasons: + curr_version = Version(last_tag) + assert curr_version.base_version.startswith( + branch + ), "Current-version has to belong to the current branch!" + next_version = Version(f"{branch}.{curr_version.micro + 1}") + print( + f"A Z-release is needed for {branch}, " + f"Prev: {last_tag}, " + f"Next: {next_version.base_version}, " + f"Reason: {','.join(reasons)}" + ) + releases.append(next_version) + else: + # Check if a Y release is needed + changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") + for change in changes.split("\n"): + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: + # We don't put Y release bumps in the commit message, check file instead. + # The 'current_version' is always the dev of the next version to release. + next_version = current_version(repo, DEFAULT_BRANCH).base_version + print(f"A new Y-release is needed! New Version: {next_version}") releases.append(next_version) - else: - # Check if a Y release is needed - changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") - for change in changes.split("\n"): - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - # We don't put Y release bumps in the commit message, check file instead - # The 'current_version' is always the next version to release - next_version = repo.git.grep( - "current_version", DEFAULT_BRANCH, "--", ".bumpversion.cfg" - ).split("=")[-1] - next_version = Version(next_version) - print( - f"A new Y-release is needed! New Version: {next_version.base_version}" - ) - releases.append(next_version) - break - - if len(releases) == 0: - print("No new releases to perform.") + break + + if len(releases) == 0: + print("No new releases to perform.") + + return 0 if __name__ == "__main__": - main() + sys.exit(main(options(), template_config())) diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index 671bade19..cf9efbe97 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,11 +5,13 @@ # # For more info visit https://github.com/pulp/plugin_template +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -20,17 +22,33 @@ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -49,18 +67,21 @@ and req.name != "pulp-python-client" ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") - ops = [op for op, ver in req.specs] - spec = str(req.specs) + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) +if __name__ == "__main__": + main() diff --git a/.ci/scripts/clean_gh_release_notes.py b/.ci/scripts/clean_gh_release_notes.py new file mode 100755 index 000000000..2bf5f4c27 --- /dev/null +++ b/.ci/scripts/clean_gh_release_notes.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# This script is running with elevated privileges from the main branch against pull requests. +# +# It cleans the input from artifacts which are used by the pulp documentation internally, +# but clutter for GitHub releases + +import sys + +NOTE = """ +> [!NOTE] +> Official changes are available on [Pulp docs]({docs_url})\ +""" + + +def main(): + plugin_name = sys.argv[1] + version_str = sys.argv[2] + docs_url = f"https://pulpproject.org/{plugin_name}/changes/#{version_str}" + note_added = False + for line in sys.stdin: + if line.endswith("\n"): + line = line[:-1] + if line.startswith("#"): + print(line.split(" {: #")[0]) + if not note_added and version_str in line: + print(NOTE.format(docs_url=docs_url)) + note_added = True + else: + print(line) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index b6c64eb4a..fbb5d59d0 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -1,4 +1,12 @@ #!/bin/env python3 +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# ] +# /// + # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use @@ -7,16 +15,21 @@ # For more info visit https://github.com/pulp/plugin_template import itertools +import json import os import re import tomllib +import urllib.request +from pathlib import Path from git import GitCommandError, Repo from packaging.version import parse as parse_version + +PYPI_PROJECT = "pulp_python" + # Read Towncrier settings -with open("pyproject.toml", "rb") as fp: - tc_settings = tomllib.load(fp)["tool"]["towncrier"] +tc_settings = tomllib.loads(Path("pyproject.toml").read_text())["tool"]["towncrier"] CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( @@ -35,7 +48,7 @@ # see help(re.split) for more info. NAME_REGEX = r".*" VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*" -VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})" +VERSION_CAPTURE_REGEX = rf"(?:YANKED )?({VERSION_REGEX})" DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" TITLE_REGEX = ( "(" @@ -75,6 +88,20 @@ def main(): branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True) branches = [ref.name for ref in branches] + changed = False + + try: + response = urllib.request.urlopen(f"https://pypi.org/pypi/{PYPI_PROJECT}/json") + pypi_record = json.loads(response.read()) + yanked_versions = { + parse_version(version): release[0]["yanked_reason"] + for version, release in pypi_record["releases"].items() + if release[0]["yanked"] is True + } + except Exception: + # If something failed, just don't mark anything as yanked. + yanked_versions = {} + with open(CHANGELOG_FILE, "r") as f: main_changelog = f.read() preamble, main_changes = split_changelog(main_changelog) @@ -95,15 +122,25 @@ def main(): if left[0] != right[0]: main_changes.append(right) + if yanked_versions: + for change in main_changes: + if change[0] in yanked_versions and "YANKED" not in change[1].split("\n")[0]: + reason = yanked_versions[change[0]] + version = str(change[0]) + change[1] = change[1].replace(version, "YANKED " + version, count=1) + if reason: + change[1] = change[1].replace("\n", f"\n\nYank reason: {reason}\n", count=1) + changed = True + new_length = len(main_changes) - if old_length < new_length: - print(f"{new_length - old_length} new versions have been added.") + if old_length < new_length or changed: + print(f"{new_length - old_length} new versions have been added (or something has changed).") with open(CHANGELOG_FILE, "w") as fp: fp.write(preamble) for change in main_changes: fp.write(change[1]) - repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE) + repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE) if __name__ == "__main__": diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py new file mode 100755 index 000000000..0c478a212 --- /dev/null +++ b/.ci/scripts/pr_labels.py @@ -0,0 +1,60 @@ +#!/bin/env python3 + +# This script is running with elevated privileges from the main branch against pull requests. + +import re +import sys +import tomllib +from pathlib import Path + +from git import Repo + + +def main(): + assert len(sys.argv) == 3 + + with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) + BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT") + ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)") + CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$") + try: + CHANGELOG_EXTS = { + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + } + except KeyError: + CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"} + + repo = Repo(".") + + base_commit = repo.commit(sys.argv[1]) + head_commit = repo.commit(sys.argv[2]) + + pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}")) + + labels = { + "multi-commit": len(pr_commits) > 1, + "cherry-pick": False, + "no-issue": False, + "no-changelog": False, + "wip": False, + } + for commit in pr_commits: + labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None + no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None + labels["no-issue"] |= no_issue + cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None + labels["cherry-pick"] |= cherry_pick + changelog_snippets = [ + k + for k in commit.stats.files + if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS + ] + labels["no-changelog"] |= not changelog_snippets + + print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v))) + print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v))) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/skip_tests.py b/.ci/scripts/skip_tests.py new file mode 100755 index 000000000..a68d000d6 --- /dev/null +++ b/.ci/scripts/skip_tests.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +skip_tests.py - Check if only documentation files were changed in a git branch + +Usage: + ./skip_tests.py + +Arguments: + git_root: The root directory of the git project + reference_branch: The branch to compare against + +Returns: + 0: Skip + 1: NoSkip + *: Error +""" + +import sys +import os +import re +import git +import textwrap +import argparse + +DOC_PATTERNS = [ + r"^docs/", + r"\.md$", + r"LICENSE.*", + r"CHANGELOG.*", + r"CHANGES.*", + r"CONTRIBUTING.*", +] + +# Exit codes +CODE_SKIP = 0 +CODE_NO_SKIP = 1 +CODE_ERROR = 2 + + +def main() -> int: + git_root, reference_branch = get_args() + changed_files = get_changed_files(git_root, reference_branch) + if not changed_files: + return CODE_SKIP + doc_files = [f for f in changed_files if is_doc_file(f)] + not_doc_files = set(changed_files) - set(doc_files) + print_changes(doc_files, not_doc_files) + if not_doc_files: + return CODE_NO_SKIP + else: + return CODE_SKIP + + +# Utils + + +def get_changed_files(git_root: str, reference_branch: str) -> list[str]: + """Get list of files changed between current branch and reference branch.""" + repo = git.Repo(git_root) + diff_index = repo.git.diff("--name-only", reference_branch).strip() + if not diff_index: + return [] + return [f.strip() for f in diff_index.split("\n") if f.strip()] + + +def is_doc_file(file_path: str) -> bool: + """Check if a file is a documentation file.""" + for pattern in DOC_PATTERNS: + if re.search(pattern, file_path): + return True + return False + + +def print_changes(doc_files: list[str], not_doc_files: list[str]) -> None: + display_doc = " \n".join(doc_files) + print(f"doc_files({len(doc_files)})") + if doc_files: + display_doc = "\n".join(doc_files) + print(textwrap.indent(display_doc, " ")) + + print(f"non_doc_files({len(not_doc_files)})") + if not_doc_files: + display_non_doc = " \n".join(not_doc_files) + print(textwrap.indent(display_non_doc, " ")) + + +def get_args() -> tuple[str, str]: + """Parse command line arguments and validate them.""" + parser = argparse.ArgumentParser(description="Check if CI can skip tests for a git branch") + parser.add_argument("git_root", help="The root directory of the git project") + parser.add_argument("reference_branch", help="The branch to compare against") + args = parser.parse_args() + git_root = os.path.abspath(args.git_root) + ref_branch = args.reference_branch + + if not os.path.exists(git_root): + raise ValueError(f"Git root directory does not exist: {git_root}") + if not os.path.isdir(git_root): + raise ValueError(f"Git root is not a directory: {git_root}") + try: + git.Repo(git_root) + except git.InvalidGitRepositoryError: + raise ValueError(f"Directory is not a git repository: {git_root}") + return git_root, ref_branch + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + print(e) + sys.exit(CODE_ERROR) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py old mode 100755 new mode 100644 index d6f22c71a..a4dc9004a --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -1,46 +1,31 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template +# This file is managed by the plugin template. +# Do not edit. +import os import re +import subprocess import sys +import tomllib +import yaml from pathlib import Path -import subprocess - -import os -import warnings from github import Github -NO_ISSUE = "[noissue]" -CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] -sha = sys.argv[1] -message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - - -KEYWORDS = ["fixes", "closes"] - -g = Github(os.environ.get("GITHUB_TOKEN")) -repo = g.get_repo("pulp/pulp_python") - - -def __check_status(issue): +def check_status(issue, repo, cherry_pick): gi = repo.get_issue(int(issue)) if gi.pull_request: sys.exit(f"Error: issue #{issue} is a pull request.") - if gi.closed_at and "cherry picked from commit" not in message: - warnings.warn( - "When backporting, use the -x flag to append a line that says " - "'(cherry picked from commit ...)' to the original commit message." + if gi.closed_at and not cherry_pick: + print("Make sure to use 'git cherry-pick -x' when backporting a change.") + print( + "If a backport of a change requires a significant amount of rewriting, " + "consider creating a new issue." ) sys.exit(f"Error: issue #{issue} is closed.") -def __check_changelog(issue): +def check_changelog(issue, CHANGELOG_EXTS): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -48,31 +33,65 @@ def __check_changelog(issue): for match in matches: if match.suffix not in CHANGELOG_EXTS: sys.exit(f"Invalid extension for changelog entry '{match}'.") - if match.suffix == ".feature" and "cherry picked from commit" in message: - sys.exit(f"Can not backport '{match}' as it is a feature.") - - -print("Checking commit message for {sha}.".format(sha=sha[0:7])) - -# validate the issue attached to the commit -regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -pattern = re.compile(regex, re.IGNORECASE) - -issues = pattern.findall(message) - -if issues: - for issue in pattern.findall(message): - __check_status(issue) - __check_changelog(issue) -else: - if NO_ISSUE in message: - print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) - elif "Merge" in message and "cherry picked from commit" in message: - pass - else: - sys.exit( - "Error: no attached issues found for {sha}. If this was intentional, add " - " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) - ) -print("Commit message for {sha} passed.".format(sha=sha[0:7])) + +def main() -> None: + TEMPLATE_CONFIG = yaml.safe_load(Path("template_config.yml").read_text()) + GITHUB_ORG = TEMPLATE_CONFIG["github_org"] + PLUGIN_NAME = TEMPLATE_CONFIG["plugin_name"] + + with Path("pyproject.toml").open("rb") as _fp: + PYPROJECT_TOML = tomllib.load(_fp) + KEYWORDS = ["fixes", "closes"] + BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"^fixup!", # This is created by 'git commit --fixup' + r"Apply suggestions from code review", # This usually comes from GitHub + ] + try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] + except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] + NOISSUE_MARKER = "[noissue]" + + sha = sys.argv[1] + message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") + + if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + + blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] + if blocking_matches: + print("Found these phrases in the commit message:") + for m in blocking_matches: + print(" - " + m.group(0)) + sys.exit("This PR is not ready for consumption.") + + g = Github(os.environ.get("GITHUB_TOKEN")) + repo = g.get_repo(f"{GITHUB_ORG}/{PLUGIN_NAME}") + + print("Checking commit message for {sha}.".format(sha=sha[0:7])) + + # validate the issue attached to the commit + issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords="|".join(KEYWORDS)) + issues = re.findall(issue_regex, message, re.IGNORECASE) + cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" + cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) + + if issues: + for issue in issues: + check_status(issue, repo, cherry_pick) + check_changelog(issue, CHANGELOG_EXTS) + + print("Commit message for {sha} passed.".format(sha=sha[0:7])) + + +if __name__ == "__main__": + main() diff --git a/.flake8 b/.flake8 index 64403dceb..9e00b3faa 100644 --- a/.flake8 +++ b/.flake8 @@ -8,13 +8,14 @@ exclude = ./docs/*,*/migrations/* per-file-ignores = */__init__.py: F401 -ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402 +ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 max-line-length = 100 # Flake8 builtin codes # -------------------- # E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed # W503: This enforces operators before line breaks which is not pep8 or black compatible. +# F824: 'nonlocal' is unused: name is never assigned in scope # Flake8-quotes extension codes # ----------------------------- diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..4bee60faf --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,15 @@ + + +### 📜 Checklist + +- [ ] Commits are cleanly separated with meaningful messages (simple features and bug fixes should be [squashed](https://pulpproject.org/pulpcore/docs/dev/guides/git/#rebasing-and-squashing) to one commit) +- [ ] A [changelog entry](https://pulpproject.org/pulpcore/docs/dev/guides/git/#changelog-update) or entries has been added for any significant changes +- [ ] Follows the [Pulp policy on AI Usage](https://pulpproject.org/help/more/governance/ai_policy/) +- [ ] (For new features) - User documentation and test coverage has been added + +See: [Pull Request Walkthrough](https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/) diff --git a/.github/template_gitref b/.github/template_gitref deleted file mode 100644 index 7150dcb43..000000000 --- a/.github/template_gitref +++ /dev/null @@ -1 +0,0 @@ -2021.08.26-345-g747bcf8 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d1fcf2e5f..988902a0f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,26 +19,24 @@ jobs: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install packaging twine wheel mkdocs jq - echo ::endgroup:: + pip install build packaging twine wheel mkdocs jq - name: "Build package" run: | - python3 setup.py sdist bdist_wheel --python-tag py3 + python3 -m build twine check dist/* - name: "Install built packages" run: | @@ -48,7 +46,7 @@ jobs: pulpcore-manager openapi --file "api.json" pulpcore-manager openapi --bindings --component "python" --file "python-api.json" - name: "Upload Package whl" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "plugin_package" path: "pulp_python/dist/" @@ -56,7 +54,7 @@ jobs: retention-days: 5 overwrite: true - name: "Upload API specs" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "api_spec" path: | @@ -75,7 +73,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - name: "Upload python client packages" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "python-client.tar" path: | @@ -84,7 +82,7 @@ jobs: retention-days: 5 overwrite: true - name: "Upload python client docs" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "python-client-docs.tar" path: | @@ -102,7 +100,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - name: "Upload Ruby client" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "ruby-client.tar" path: | diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml deleted file mode 100644 index 9450eb09a..000000000 --- a/.github/workflows/changelog.yml +++ /dev/null @@ -1,58 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - ---- -name: "Python changelog update" -on: - push: - branches: - - "main" - paths: - - "CHANGES.rst" - - "CHANGES.md" - workflow_dispatch: - -jobs: - - update-changelog: - runs-on: "ubuntu-latest" - strategy: - fail-fast: false - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install -r doc_requirements.txt - echo ::endgroup:: - - - name: "Fake api schema" - run: | - mkdir -p docs/_build/html - echo "{}" > docs/_build/html/api.json - mkdir -p docs/_static - echo "{}" > docs/_static/api.json - - name: "Build Docs" - run: | - make diagrams html - working-directory: "./docs" - env: - PULP_CONTENT_ORIGIN: "http://localhost/" - - - name: "Publish changlog to pulpproject.org" - run: | - .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} - env: - PULP_DOCS_KEY: "${{ secrets.PULP_DOCS_KEY }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17800617c..2965afca1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,18 +21,16 @@ jobs: check-commits: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install requests pygithub - echo ::endgroup:: + pip install requests pygithub pyyaml - name: "Check commit message" if: github.event_name == 'pull_request' env: @@ -42,24 +40,78 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" run: | .github/workflows/scripts/check_commit.sh - - name: "Verify requirements files" + + check-changes: + runs-on: "ubuntu-latest" + outputs: + run_tests: "${{ steps.check.outputs.run_tests }}" + run_docs: "${{ steps.check.outputs.run_docs }}" + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 0 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.12" + + - name: "Install python dependencies" run: | - python .ci/scripts/check_requirements.py + pip install gitpython + + - name: "Analyze changed files" + id: "check" + shell: "bash" + run: | + # We only test docs on the default branch (usually main) + if [[ "${{ github.base_ref }}" == *"main" ]]; then + echo "run_docs=1" >> $GITHUB_OUTPUT + else + echo "run_docs=0" >> $GITHUB_OUTPUT + fi + + set +e + BASE_REF=${{ github.event.pull_request.base.sha }} + echo "Checking against:" + git name-rev $BASE_REF + python3 .ci/scripts/skip_tests.py . $BASE_REF + exit_code=$? + if [ $exit_code -ne 0 ] && [ $exit_code -ne 1 ]; then + echo "Error: skip_tests.py returned unexpected exit code $exit_code" + exit $exit_code + fi + echo "run_tests=$exit_code" >> $GITHUB_OUTPUT + + docs: + needs: "check-changes" + uses: "./.github/workflows/docs.yml" + with: + run_docs: "${{ needs.check-changes.outputs.run_docs }}" lint: uses: "./.github/workflows/lint.yml" + sanity: + uses: "./.github/workflows/sanity.yml" + build: - needs: "lint" + needs: + - "check-changes" + - "lint" + if: "needs.check-changes.outputs.run_tests == '1'" uses: "./.github/workflows/build.yml" test: needs: "build" uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] deprecations: runs-on: "ubuntu-latest" - if: github.base_ref == 'main' + if: "github.base_ref == 'main'" needs: "test" steps: - name: "Create working directory" @@ -67,7 +119,7 @@ jobs: mkdir -p "pulp_python" working-directory: "." - name: "Download Deprecations" - uses: actions/download-artifact@v4 + uses: "actions/download-artifact@v8" with: pattern: "deprecations-*" path: "pulp_python" @@ -81,14 +133,42 @@ jobs: # This is a dummy dependent task to have a single entry for the branch protection rules. runs-on: "ubuntu-latest" needs: + - "check-changes" - "check-commits" - "lint" - "test" + - "docs" + - "sanity" if: "always()" steps: - name: "Collect needed jobs results" working-directory: "." run: | - echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result' - echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0' + RUN_TESTS=${{ needs.check-changes.outputs.run_tests }} + RUN_DOCS=${{ needs.check-changes.outputs.run_docs }} + + check_jobs() { + local filter="$1" + local needs_json='${{toJson(needs)}}' + # output failed jobs after filter + echo "$needs_json" | jq -r "to_entries[]|select($filter)|select(.value.result!=\"success\")|.key + \": \" + .value.result" + # fails if not all selected jobs passed + echo "$needs_json" | jq -e "to_entries|map(select($filter))|map(select(.value.result!=\"success\"))|length == 0" + } + + if [ "$RUN_TESTS" == "1" ] && [ "$RUN_DOCS" == "1" ]; then + FILTERS="true" # check all jobs + elif [ "$RUN_TESTS" == "1" ] && [ "$RUN_DOCS" == "0" ]; then + echo "Skipping docs: running on non-default branch" + FILTERS='.key != "docs"' + elif [ "$RUN_TESTS" == "0" ] && [ "$RUN_DOCS" == "1" ]; then + echo "Skipping tests: only doc changes" + FILTERS='.key != "lint" and .key != "test"' + else # RUN_TESTS=0, RUN_DOCS=0 + echo "What is this PR doing??" + FILTERS='.key != "lint" and .key != "test" and .key != "docs"' + fi + + check_jobs "$FILTERS" echo "CI says: Looks good!" +... diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 899147c47..22e5d99b8 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,39 +4,42 @@ # './plugin-template --github pulp_python' to update this file. # # For more info visit https://github.com/pulp/plugin_template +--- name: "Python CodeQL" on: workflow_dispatch: schedule: - - cron: '37 1 * * 6' + - cron: "37 1 * * 6" concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: analyze: - name: Analyze - runs-on: ubuntu-latest + name: "Analyze" + runs-on: "ubuntu-latest" permissions: - actions: read - contents: read - security-events: write + actions: "read" + contents: "read" + security-events: "write" strategy: fail-fast: false matrix: - language: [ 'python' ] + language: + - "python" steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: "Checkout repository" + uses: "actions/checkout@v6" - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + - name: "Initialize CodeQL" + uses: "github/codeql-action/init@v4" with: - languages: ${{ matrix.language }} + languages: "${{ matrix.language }}" - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: "Perform CodeQL Analysis" + uses: "github/codeql-action/analyze@v4" +... diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index b207c880f..6b57c7854 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Create New Release Branch +name: "Create New Release Branch" on: workflow_dispatch: @@ -15,26 +15,33 @@ env: jobs: create-branch: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false + permissions: + contents: "write" + steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + repository: "pulp/plugin_template" + path: "plugin_template" + + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install bump2version jinja2 pyyaml packaging - echo ::endgroup:: + pip install bump-my-version packaging -r plugin_template/requirements.txt - name: "Setting secrets" working-directory: "pulp_python" @@ -43,12 +50,12 @@ jobs: env: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: Determine new branch name - working-directory: pulp_python + - name: "Determine new branch name" + working-directory: "pulp_python" run: | # Just to be sure... git checkout main - NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')" + NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')" if [ -z "$NEW_BRANCH" ] then echo Could not determine the new branch name. @@ -56,51 +63,43 @@ jobs: fi echo "NEW_BRANCH=${NEW_BRANCH}" >> "$GITHUB_ENV" - - name: Create release branch - working-directory: pulp_python + - name: "Create release branch" + working-directory: "pulp_python" run: | git branch "${NEW_BRANCH}" - - name: Bump version on main branch - working-directory: pulp_python + - name: "Bump version on main branch" + working-directory: "pulp_python" run: | - bump2version --no-commit minor + bump-my-version bump --no-commit minor - - name: Remove entries from CHANGES directory - working-directory: pulp_python + - name: "Remove entries from CHANGES directory" + working-directory: "pulp_python" run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + - - name: Checkout plugin template - uses: actions/checkout@v4 - with: - repository: pulp/plugin_template - path: plugin_template - fetch-depth: 0 - - - name: Update CI branches in template_config - working-directory: plugin_template + - name: "Update CI branches in template_config" + working-directory: "plugin_template" run: | python3 ./plugin-template pulp_python --github --latest-release-branch "${NEW_BRANCH}" git add -A - - name: Make a PR with version bump and without CHANGES/* - uses: peter-evans/create-pull-request@v6 + - name: "Make a PR with version bump and without CHANGES/*" + uses: "peter-evans/create-pull-request@v8" with: - path: pulp_python - token: ${{ secrets.RELEASE_TOKEN }} - committer: pulpbot - author: pulpbot - branch: minor-version-bump - base: main - title: Bump minor version - body: '[noissue]' + path: "pulp_python" + token: "${{ secrets.RELEASE_TOKEN }}" + committer: "pulpbot " + author: "pulpbot " + branch: "minor-version-bump" + base: "main" + title: "Bump minor version" commit-message: | Bump minor version - [noissue] delete-branch: true - - name: Push release branch - working-directory: pulp_python + - name: "Push release branch" + working-directory: "pulp_python" run: | git push origin "${NEW_BRANCH}" +... diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..c897fcac2 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,43 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Docs CI" +on: + workflow_call: + inputs: + run_docs: + description: "Whether to run docs jobs" + required: true + type: "string" + +jobs: + changelog: + runs-on: "ubuntu-latest" + defaults: + run: + working-directory: "pulp_python" + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + - uses: "actions/setup-python@v6" + with: + python-version: "3.12" + - name: "Install python dependencies" + run: | + pip install towncrier + - name: "Build changelog" + run: | + towncrier build --yes --version 4.0.0.ci + docs: + if: "${{ inputs.run_docs == '1' }}" + uses: "pulp/pulp-docs/.github/workflows/docs-ci.yml@main" + with: + pulpdocs_ref: "main" +... diff --git a/.github/workflows/kanban.yml b/.github/workflows/kanban.yml deleted file mode 100644 index 370f4bee7..000000000 --- a/.github/workflows/kanban.yml +++ /dev/null @@ -1,103 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -# Manage issues in a project board using https://github.com/leonsteinhaeuser/project-beta-automations - ---- -name: Kanban -on: - pull_request_target: - issues: - types: - - labeled - - reopened - - assigned - - closed - -env: - free_to_take: Free to take - in_progress: In Progress - needs_review: Needs review - done: Done - -jobs: - # only prio-list labeled items should be added to the board - add-to-project-board: - if: github.event_name == 'issues' && contains(github.event.issue.labels.*.name, 'prio-list') && contains(fromJson('["labeled", "reopened"]'), github.event.action) - runs-on: ubuntu-latest - steps: - - name: Add issue to Free-to-take list - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.free_to_take }} # Target status - - move-to-inprogress: - if: github.event_name == 'issues' && github.event.action == 'assigned' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the In Progress column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.in_progress }} # Target status - - find-linked-issues: - if: github.event_name == 'pull_request_target' - runs-on: ubuntu-latest - name: Find issues linked to a PR - outputs: - linked-issues: ${{ steps.linked-issues.outputs.issues }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Get Linked Issues Action - uses: kin/gh-action-get-linked-issues@v1.0 - id: linked-issues - with: - access-token: ${{ secrets.RELEASE_TOKEN }} - - move-to-needs-review: - if: github.event_name == 'pull_request_target' && contains(fromJson(needs.find-linked-issues.outputs.linked-issues).*.issue.state, 'open') - runs-on: ubuntu-latest - name: Move linked issues to Needs Review - needs: find-linked-issues - strategy: - max-parallel: 3 - matrix: - issues: ${{ fromJSON(needs.find-linked-issues.outputs.linked-issues) }} - steps: - - name: Move to Needs Review - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ matrix.issues.issue.node_id }} - operation_mode: status - status_value: ${{ env.needs_review }} # Target status - - move-to-done: - if: github.event_name == 'issues' && github.event.action == 'closed' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the Done column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.done }} # Target status diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 701d990ef..d07a39a08 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,41 +16,36 @@ defaults: jobs: lint: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install -r lint_requirements.txt - echo ::endgroup:: - - name: Lint workflow files + - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows # Lint code. - - name: Run flake8 - run: flake8 - - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" - - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + - name: "Run flake8" + run: | + flake8 - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh +... diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5b9b39ac4..ed8149ce9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ on: schedule: # * is a special character in YAML so you have to quote this string # runs at 3:00 UTC daily - - cron: '00 3 * * *' + - cron: "00 3 * * *" workflow_dispatch: defaults: @@ -29,91 +29,51 @@ jobs: test: needs: "build" uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] changelog: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: - python-version: "3.11" + python-version: "3.13" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install gitpython packaging toml - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Collect changes from all branches - run: python .ci/scripts/collect_changes.py + - name: "Collect changes from all branches" + run: | + python .ci/scripts/collect_changes.py - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 + - name: "Create Pull Request" + uses: "peter-evans/create-pull-request@v8" + id: "create_pr_changelog" with: - token: ${{ secrets.RELEASE_TOKEN }} + token: "${{ secrets.RELEASE_TOKEN }}" title: "Update Changelog" body: "" branch: "changelog/update" delete-branch: true path: "pulp_python" - - publish: - runs-on: ubuntu-latest - needs: test - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_python" - - - uses: actions/download-artifact@v4 - with: - name: "plugin_package" - path: "pulp_python/dist/" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1' - echo ::endgroup:: - - - name: "Set environment variables" - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Download built docs - uses: actions/download-artifact@v4 - with: - name: "docs.tar" - path: "pulp_python" - - - name: Download Python client docs - uses: actions/download-artifact@v4 - with: - name: "python-client-docs.tar" - path: "pulp_python" - - - name: "Setting secrets" + - name: "Mark PR automerge" + working-directory: "pulp_python" run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + gh pr merge --rebase --auto "${{ steps.create_pr_changelog.outputs.pull-request-number }}" + if: "steps.create_pr_changelog.outputs.pull-request-number" env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar -C ./docs - .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true +... diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index f2a44ab6e..d80548961 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -6,57 +6,68 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Python PR static checks +name: "Python PR static checks" on: pull_request_target: - types: [opened, synchronize, reopened] + types: + - "opened" + - "synchronize" + - "reopened" + branches: + - "main" + - "[0-9]+.[0-9]+" # This workflow runs with elevated permissions. # Do not even think about running a single bit of code from the PR. # Static analysis should be fine however. concurrency: - group: ${{ github.event.pull_request.number }}-${{ github.workflow }} + group: "${{ github.event.pull_request.number }}-${{ github.workflow }}" cancel-in-progress: true jobs: - single_commit: - runs-on: ubuntu-latest - name: Label multiple commit PR + apply_labels: + runs-on: "ubuntu-latest" + name: "Label PR" permissions: - pull-requests: write + pull-requests: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 - - name: Commit Count Check + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + - name: "Determine PR labels" run: | + pip install GitPython==3.1.42 git fetch origin ${{ github.event.pull_request.head.sha }} - echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV" - - uses: actions/github-script@v7 + python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" + - uses: "actions/github-script@v8" + name: "Apply PR Labels" with: script: | - const labelName = "multi-commit"; - const { COMMIT_COUNT } = process.env; + const { ADD_LABELS, REMOVE_LABELS } = process.env; - if (COMMIT_COUNT == 1) - { - try { - await github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - }); - } catch(err) { + if (REMOVE_LABELS.length) { + for await (const labelName of REMOVE_LABELS.split(",")) { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } } } - else - { + if (ADD_LABELS.length) { await github.rest.issues.addLabels({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - labels: [labelName], + labels: ADD_LABELS.split(","), }); } +... diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1c225e732..6d0d3aa8c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,175 +19,35 @@ defaults: jobs: build: uses: "./.github/workflows/build.yml" - - build-bindings-docs: - needs: - - "build" - runs-on: "ubuntu-latest" - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: "docs" - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_python" - - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - repository: "pulp/pulp-openapi-generator" - path: "pulp-openapi-generator" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - uses: "actions/download-artifact@v4" - with: - name: "plugin_package" - path: "pulp_python/dist/" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: "Before Install" - run: | - .github/workflows/scripts/before_install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install" - run: | - .github/workflows/scripts/install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - - name: "Upload python client packages" - uses: "actions/upload-artifact@v4" - with: - name: "python-client.tar" - path: | - pulp_python/python-python-client.tar - if-no-files-found: "error" - overwrite: true - - - name: "Upload python client docs" - uses: "actions/upload-artifact@v4" - with: - name: "python-client-docs.tar" - path: | - pulp_python/python-python-client-docs.tar - if-no-files-found: "error" - overwrite: true - - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v4" - with: - name: "ruby-client.tar" - path: | - pulp_python/python-ruby-client.tar - if-no-files-found: "error" - overwrite: true - - name: Build docs - run: | - export DJANGO_SETTINGS_MODULE=pulpcore.app.settings - export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py - make -C docs/ PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" diagrams html - tar -cvf docs/docs.tar docs/_build - - - name: "Upload built docs" - uses: actions/upload-artifact@v4 - with: - name: "docs.tar" - path: "pulp_python/docs/docs.tar" - if-no-files-found: "error" - overwrite: true - - - name: "Logs" - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" publish-package: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + - "build" + environment: + name: "pypi" + url: "https://pypi.org/p/pulp-python" + permissions: + id-token: "write" steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_python" - - uses: "actions/download-artifact@v4" with: name: "plugin_package" - path: "pulp_python/dist/" + path: "dist/" - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Deploy plugin to pypi" - run: | - .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.ref_name }} + - name: "Publish package to PyPI" + uses: pypa/gh-action-pypi-publish@release/v1 publish-python-bindings: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + - "build" + environment: + name: "pypi" + permissions: + id-token: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" @@ -202,35 +62,21 @@ jobs: run: | tar -xvf python-python-client.tar - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: "Publish client to pypi" - run: | - bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.ref_name }} + uses: "pypa/gh-action-pypi-publish@release/v1" + with: + packages-dir: "pulp_python/dist/" publish-ruby-bindings: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + - "build" + environment: + name: "rubygems" + permissions: + id-token: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" @@ -249,83 +95,65 @@ jobs: with: ruby-version: "2.6" - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" + - name: "Set RubyGems Credentials" + uses: "rubygems/configure-rubygems-credentials@v1.0.0" - - name: "Publish client to rubygems" + - name: "Publish client to RubyGems" run: | - bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} - publish-docs: + gem push "pulp_python_client-${{ github.ref_name }}.gem" + + create-gh-release: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" + - "publish-package" + - "publish-python-bindings" + - "publish-ruby-bindings" + + permissions: + contents: "write" env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + TAG_NAME: "${{ github.ref_name }}" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: - fetch-depth: 1 + fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - - name: "Install python dependencies" + - name: "Install towncrier" run: | - echo ::group::PYDEPS - pip install 'packaging~=21.3' requests - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Download built docs" - uses: "actions/download-artifact@v4" - with: - name: "docs.tar" - path: "pulp_python/" + pip install towncrier - - name: "Download Python client docs" - uses: "actions/download-artifact@v4" - with: - name: "python-client-docs.tar" - path: "pulp_python/" - - - name: "Publish docs to pulpproject.org" + - name: "Get release notes" + id: "get_release_notes" + shell: "bash" run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }} + # The last commit before the release commit contains the release CHANGES fragments + git checkout "${TAG_NAME}~" + NOTES=$(towncrier build --draft --version $TAG_NAME | .ci/scripts/clean_gh_release_notes.py pulp_python $TAG_NAME) + echo "body<> $GITHUB_OUTPUT + echo "$NOTES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - create-gh-release: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-package" - - "publish-python-bindings" - - "publish-ruby-bindings" - - "publish-docs" - - steps: - name: "Create release on GitHub" - uses: "actions/github-script@v7" + uses: "actions/github-script@v8" env: - TAG_NAME: "${{ github.ref_name }}" + RELEASE_BODY: "${{ steps.get_release_notes.outputs.body }}" with: script: | - const { TAG_NAME } = process.env; + const { TAG_NAME, RELEASE_BODY } = process.env; await github.rest.repos.createRelease({ owner: context.repo.owner, repo: context.repo.repo, tag_name: TAG_NAME, + body: RELEASE_BODY, make_latest: "legacy", }); +... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c362a0a37..e056bcfaa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,21 +22,19 @@ jobs: fail-fast: false steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" token: ${{ secrets.RELEASE_TOKEN }} - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install bump2version towncrier - echo ::endgroup:: + pip install bump-my-version towncrier - name: "Configure Git with pulpbot name and email" run: | @@ -58,3 +56,4 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" +... diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml new file mode 100644 index 000000000..223f07cbd --- /dev/null +++ b/.github/workflows/sanity.yml @@ -0,0 +1,54 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + + +# This file describes checks that should prevent a premature merge, +# but still let the tests run for demonstrations or experiments. +--- +name: "Sanity" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + sanity: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + pip install -r lint_requirements.txt + + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest + + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py + + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh +... diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 5abc9c984..b33df1d5c 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -7,69 +7,126 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script prepares the scenario definition in the .ci/ansible/vars/main.yaml file. +# +# It requires the following environment: +# TEST - The name of the scenario to prepare. +# +# It may also dump the {lower,upper}bounds_constraints.txt for the specific scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -mveuo pipefail +if [ -f .github/workflows/scripts/pre_before_install.sh ]; then + source .github/workflows/scripts/pre_before_install.sh +fi -if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ] -then - BRANCH_BUILD=0 -else - BRANCH_BUILD=1 - BRANCH="${GITHUB_REF##refs/heads/}" +COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" +COMPONENT_SOURCE="./pulp_python/dist/pulp_python-${COMPONENT_VERSION}-py3-none-any.whl" +if [ "$TEST" = "s3" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" fi -if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ] -then - TAG_BUILD=0 -else - TAG_BUILD=1 - BRANCH="${GITHUB_REF##refs/tags/}" +if [ "$TEST" = "azure" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]" fi -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export COMMIT_MSG +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt +fi +if [[ "$TEST" = "lowerbounds" ]]; then + python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt +fi -COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) +export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") -mkdir .ci/ansible/vars || true -echo "---" > .ci/ansible/vars/main.yaml -echo "legacy_component_name: pulp_python" >> .ci/ansible/vars/main.yaml -echo "component_name: python" >> .ci/ansible/vars/main.yaml -echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml +echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" -export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh -export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh +# Compose the scenario definition. +mkdir -p .ci/ansible/vars -if [ -f $PRE_BEFORE_INSTALL ]; then - source $PRE_BEFORE_INSTALL -fi +cat > .ci/ansible/vars/main.yaml << VARSYAML +--- +scenario: "${TEST}" +plugin_name: "pulp_python" +legacy_component_name: "pulp_python" +component_name: "python" +component_version: "${COMPONENT_VERSION}" +pulp_env: {} +pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} +pulp_scheme: "https" +api_root: "${PULP_API_ROOT}" +image: + name: "pulp" + tag: "ci_build" + ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest" + source: "${COMPONENT_SOURCE}" + ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) + upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) + lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) + webserver_snippet: $(test -f pulp_python/app/webserver_snippets/nginx.conf && echo -n true || echo -n false ) +extra_files: + - origin: "pulp_python" + destination: "pulp_python" +services: + - name: "pulp" + image: "pulp:ci_build" + volumes: + - "./settings:/etc/pulp" + - "../../../pulp-openapi-generator:/root/pulp-openapi-generator" + env: + PULP_WORKERS: "4" + PULP_HTTPS: "true" +VARSYAML -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 +if [ "$TEST" = "s3" ]; then + MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA + MINIO_SECRET_KEY=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 + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "minio" + image: "minio/minio" + env: + MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}" + MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}" + command: "server /data" +s3_test: true +minio_access_key: "${MINIO_ACCESS_KEY}" +minio_secret_key: "${MINIO_SECRET_KEY}" +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true} +pulp_scenario_env: {} +VARSYAML fi -if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] -then - echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml +if [ "$TEST" = "azure" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-azurite" + image: "mcr.microsoft.com/azure-storage/azurite" + command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0" +azure_test: true +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true} +pulp_scenario_env: {} +VARSYAML fi -for i in {1..3} -do - ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3 -done -if [[ $s -gt 0 ]] -then - echo "Failed to install amazon.aws" - exit $s +if [ "$TEST" = "gcp" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-gcp" + image: "fsouza/fake-gcs-server" + volumes: + - "storage_data:/etc/pulp" + command: " -scheme http" +gcp_test: true +pulp_scenario_settings: null +pulp_scenario_env: {} +VARSYAML fi -if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt - sed -i 's/\[.*\]//g' lowerbounds_constraints.txt -fi +cat >> .ci/ansible/vars/main.yaml << VARSYAML +... +VARSYAML +cat .ci/ansible/vars/main.yaml -if [ -f $POST_BEFORE_INSTALL ]; then - source $POST_BEFORE_INSTALL +if [ -f .github/workflows/scripts/post_before_install.sh ]; then + source .github/workflows/scripts/post_before_install.sh fi diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 5c0072c42..13809a87a 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -7,44 +7,46 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script dumps some files to help understand the setup of the test scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -euv - source .github/workflows/scripts/utils.sh -export PRE_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/pre_before_script.sh -export POST_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/post_before_script.sh - -if [[ -f $PRE_BEFORE_SCRIPT ]]; then - source $PRE_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/pre_before_script.sh ]]; then + source .github/workflows/scripts/pre_before_script.sh fi -# Developers should be able to reproduce the containers with this config -echo "CI vars:" -tail -v -n +1 .ci/ansible/vars/main.yaml - # Developers often want to know the final pulp config -echo "PULP CONFIG:" -tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json +echo +echo "# Pulp config:" +tail -v -n +1 .ci/ansible/settings/settings.* -echo "Containerfile:" +echo +echo "# Containerfile:" tail -v -n +1 .ci/ansible/Containerfile -# Needed for some functional tests -cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" -cmd_prefix bash -c "usermod -a -G wheel pulp" +echo +echo "# Constraints Files:" +# They need not even exist. +tail -v -n +1 ../*/*constraints.txt || true -if [[ "${REDIS_DISABLED:-false}" == true ]]; then - cmd_prefix bash -c "s6-rc -d change redis" - echo "The Redis service was disabled for $TEST" -fi +echo +echo "# pip list outside the container" +pip list + +echo +echo "# pip list inside the container" +cmd_prefix bash -c "pip3 list" + +echo +echo "# State of the containers" +docker ps -a -if [[ -f $POST_BEFORE_SCRIPT ]]; then - source $POST_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/post_before_script.sh ]]; then + source .github/workflows/scripts/post_before_script.sh fi -# Lots of plugins try to use this path, and throw warnings if they cannot access it. -cmd_prefix mkdir /.pytest_cache -cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh index 5bf6e16d3..9eb0def9f 100755 --- a/.github/workflows/scripts/build_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -18,15 +18,13 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pushd ../pulp-openapi-generator rm -rf "pulp_python-client" -# We need to copy that over to be visible in the container. -cp "../pulp_python/python-api.json" . -./gen-client.sh "python-api.json" "python" python "pulp_python" +./gen-client.sh "../pulp_python/python-api.json" "python" python "pulp_python" pushd pulp_python-client -python setup.py sdist bdist_wheel --python-tag py3 +python -m build twine check "dist/pulp_python_client-"*"-py3-none-any.whl" -twine check "dist/pulp_python-client-"*".tar.gz" +twine check "dist/pulp_python_client-"*".tar.gz" tar cvf "../../pulp_python/python-python-client.tar" ./dist diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh index a4abec1d3..6b433edd2 100755 --- a/.github/workflows/scripts/build_ruby_client.sh +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -18,15 +18,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pushd ../pulp-openapi-generator rm -rf "pulp_python-client" -# We need to copy that over to be visible in the container. -#cp "../pulp_python/python-api.json" . -#./gen-client.sh "python-api.json" "python" ruby "pulp_python" - -# ------------- -# The generator still needs to have it called api.json at this time... -cp "../pulp_python/api.json" . -./gen-client.sh "api.json" "python" ruby "pulp_python" -# ------------- +./gen-client.sh "../pulp_python/python-api.json" "python" ruby "pulp_python" pushd pulp_python-client gem build pulp_python_client diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index e6b6d5e34..400e05423 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -15,8 +15,4 @@ set -euv for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') do python3 .ci/scripts/validate_commit_message.py "$SHA" - VALUE=$? - if [ "$VALUE" -gt 0 ]; then - exit $VALUE - fi done diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py deleted file mode 100755 index 61ae6e478..000000000 --- a/.github/workflows/scripts/docs-publisher.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import subprocess -import os -import re -from shutil import rmtree -import tempfile -import requests -import json -from packaging import version - -WORKING_DIR = os.environ["WORKSPACE"] - -VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" -RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" - -USERNAME = "doc_builder_pulp_python" -HOSTNAME = "8.43.85.236" - -SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_python/" - - -def make_directory_with_rsync(remote_paths_list): - """ - Ensure the remote directory path exists. - - :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the - remote. - :type remote_paths_list: a list of strings, with each string representing a directory. - """ - try: - tempdir_path = tempfile.mkdtemp() - cwd = os.getcwd() - os.chdir(tempdir_path) - os.makedirs(os.sep.join(remote_paths_list)) - remote_path_arg = "%s@%s:%s%s" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - remote_paths_list[0], - ) - local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep - rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command) - if exit_code != 0: - raise RuntimeError("An error occurred while creating remote directories.") - finally: - rmtree(tempdir_path) - os.chdir(cwd) - - -def ensure_dir(target_dir, clean=True): - """ - Ensure that the directory specified exists and is empty. - - By default this will delete the directory if it already exists - - :param target_dir: The directory to process - :type target_dir: str - :param clean: Whether or not the directory should be removed and recreated - :type clean: bool - """ - if clean: - rmtree(target_dir, ignore_errors=True) - try: - os.makedirs(target_dir) - except OSError: - pass - - -def main(): - """ - Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--build-type", required=True, help="Build type: nightly, tag or changelog." - ) - parser.add_argument("--branch", required=True, help="Branch or tag name.") - opts = parser.parse_args() - - build_type = opts.build_type - - branch = opts.branch - - publish_at_root = False - - # rsync the docs - print("rsync the docs") - docs_directory = os.sep.join([WORKING_DIR, "docs"]) - local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep - if build_type == "nightly": - # This is a nightly build - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - make_directory_with_rsync(["en", branch, build_type]) - rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: - # Only publish docs at the root if this is the latest version - r = requests.get("https://pypi.org/pypi/pulp-python/json") - latest_version = version.parse(json.loads(r.text)["info"]["version"]) - docs_version = version.parse(branch) - # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. - # It doesn't prevent incorrect docs being published at root if 2 releases are done close - # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately - # after. - if docs_version >= latest_version: - publish_at_root = True - # Post releases should use the x.y.z part of the version string to form a path - if "post" in branch: - branch = ".".join(branch.split(".")[:-1]) - - # This is a GA build. - # publish to the root of docs.pulpproject.org - if publish_at_root: - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "en", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - # This is a pre-release - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "nightly", - "--exclude", - "testing", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "changelog": - if branch != "main": - raise RuntimeError("Can only publish CHANGELOG from main") - # Publish the CHANGELOG from main branch at the root directory - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - changelog_local_path_arg = os.path.join(local_path_arg, "changes.html") - rsync_command = [ - "rsync", - "-vzh", - "--omit-dir-times", - changelog_local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - raise RuntimeError("Build type must be either 'nightly', 'tag' or 'changelog'.") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 7f422676c..9df47ccde 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -7,133 +7,43 @@ # # For more info visit https://github.com/pulp/plugin_template +set -euv + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. REPO_ROOT="$PWD" -set -euv - source .github/workflows/scripts/utils.sh -PLUGIN_VERSION="$(sed -n -e 's/^\s*current_version\s*=\s*//p' .bumpversion.cfg | python -c 'from packaging.version import Version; print(Version(input()))')" -PLUGIN_SOURCE="./pulp_python/dist/pulp_python-${PLUGIN_VERSION}-py3-none-any.whl" - -export PULP_API_ROOT="/pulp/" - PIP_REQUIREMENTS=("pulp-cli") -if [[ "$TEST" = "docs" || "$TEST" = "publish" ]] -then - PIP_REQUIREMENTS+=("-r" "doc_requirements.txt") - git clone https://github.com/pulp/pulpcore.git ../pulpcore - PIP_REQUIREMENTS+=("psycopg2-binary" "-r" "../pulpcore/doc_requirements.txt") -fi +# This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} -if [[ "$TEST" != "docs" ]] +if [[ "$TEST" = "s3" ]]; then +for i in {1..3} +do + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 +done +if [[ $s -gt 0 ]] then - PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" - git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -fi - -cd .ci/ansible/ -if [ "$TEST" = "s3" ]; then - PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[s3]" + echo "Failed to install amazon.aws" + exit $s fi -if [ "$TEST" = "azure" ]; then - PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[azure]" fi -cat >> vars/main.yaml << VARSYAML -image: - name: pulp - tag: "ci_build" -plugins: - - name: pulp_python - source: "${PLUGIN_SOURCE}" -VARSYAML -if [[ -f ../../ci_requirements.txt ]]; then - cat >> vars/main.yaml << VARSYAML - ci_requirements: true -VARSYAML -fi -if [ "$TEST" = "lowerbounds" ]; then - cat >> vars/main.yaml << VARSYAML - lowerbounds: true -VARSYAML -fi - -cat >> vars/main.yaml << VARSYAML -services: - - name: pulp - image: "pulp:ci_build" - volumes: - - ./settings:/etc/pulp - - ./ssh:/keys/ - - ~/.config:/var/lib/pulp/.config - - ../../../pulp-openapi-generator:/root/pulp-openapi-generator - env: - PULP_WORKERS: "4" - PULP_HTTPS: "true" -VARSYAML - -cat >> vars/main.yaml << VARSYAML -pulp_env: {} -pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} -pulp_scheme: https -pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest -VARSYAML - -if [ "$TEST" = "s3" ]; then - export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA - export MINIO_SECRET_KEY=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 - sed -i -e '/^services:/a \ - - name: minio\ - image: minio/minio\ - env:\ - MINIO_ACCESS_KEY: "'$MINIO_ACCESS_KEY'"\ - MINIO_SECRET_KEY: "'$MINIO_SECRET_KEY'"\ - command: "server /data"' vars/main.yaml - sed -i -e '$a s3_test: true\ -minio_access_key: "'$MINIO_ACCESS_KEY'"\ -minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"domain_enabled": true}\ -pulp_scenario_env: {}\ -' vars/main.yaml - export PULP_API_ROOT="/rerouted/djnd/" -fi - -if [ "$TEST" = "azure" ]; then - sed -i -e '/^services:/a \ - - name: ci-azurite\ - image: mcr.microsoft.com/azure-storage/azurite\ - volumes:\ - - ./azurite:/etc/pulp\ - command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml - sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"domain_enabled": true}\ -pulp_scenario_env: {}\ -' vars/main.yaml -fi - -echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" +# Check out the pulp-cli branch matching the installed version. +PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" +git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -if [ "${PULP_API_ROOT:-}" ]; then - sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml -fi +cd .ci/ansible/ -pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" --username "admin" --password "password" -if [[ "$TEST" != "docs" ]]; then - cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" -fi +pulp config create --base-url https://pulp --api-root "${PULP_API_ROOT}" --username "admin" --password "password" +cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml ansible-playbook start_container.yaml -# .config needs to be accessible by the pulp user in the container, but some -# files will likely be modified on the host by post/pre scripts. -chmod 777 ~/.config/pulp_smash/ -chmod 666 ~/.config/pulp_smash/settings.json # Plugins often write to ~/.config/pulp/cli.toml from the host chmod 777 ~/.config/pulp chmod 666 ~/.config/pulp/cli.toml @@ -159,6 +69,10 @@ if [[ "$TEST" = "azure" ]]; then az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING fi -echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" -echo ::endgroup:: +# Needed for some functional tests +cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" +cmd_prefix bash -c "usermod -a -G wheel pulp" + +# Lots of plugins try to use this path, and throw warnings if they cannot access it. +cmd_prefix mkdir /.pytest_cache +cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh deleted file mode 100755 index 8c1fe8f02..000000000 --- a/.github/workflows/scripts/install_python_client.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -source .github/workflows/scripts/utils.sh - -PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "python" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - -pushd ../pulp-openapi-generator -rm -rf pulp_python-client - -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=python" - USE_LOCAL_API_JSON=1 ./generate.sh pulp_python python "$VERSION" -else - ./generate.sh pulp_python python "$VERSION" -fi - -pushd pulp_python-client -python setup.py sdist bdist_wheel --python-tag py3 - -twine check "dist/pulp_python_client-$VERSION-py3-none-any.whl" -twine check "dist/pulp_python-client-$VERSION.tar.gz" - -cmd_prefix pip3 install "/root/pulp-openapi-generator/pulp_python-client/dist/pulp_python_client-${VERSION}-py3-none-any.whl" -tar cvf ../../pulp_python/python-python-client.tar ./dist - -find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; -find ./docs/* -exec sed -i 's/README//g' {} \; -cp README.md docs/index.md -sed -i 's/docs\///g' docs/index.md -find ./docs/* -exec sed -i 's/\.md//g' {} \; - -cat >> mkdocs.yml << DOCSYAML ---- -site_name: PulpPython Client -site_description: Python bindings -site_author: Pulp Team -site_url: https://docs.pulpproject.org/pulp_python_client/ -repo_name: pulp/pulp_python -repo_url: https://github.com/pulp/pulp_python -theme: readthedocs -DOCSYAML - -# Building the bindings docs -mkdocs build - -# Pack the built site. -tar cvf ../../pulp_python/python-python-client-docs.tar ./site -popd -popd diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh deleted file mode 100755 index 7ab198a59..000000000 --- a/.github/workflows/scripts/install_ruby_client.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -source .github/workflows/scripts/utils.sh - -PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "python" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - -pushd ../pulp-openapi-generator -rm -rf pulp_python-client - -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=python" - USE_LOCAL_API_JSON=1 ./generate.sh pulp_python ruby "$VERSION" -else - ./generate.sh pulp_python ruby "$VERSION" -fi - -pushd pulp_python-client -gem build pulp_python_client -gem install --both "./pulp_python_client-$VERSION.gem" -tar cvf ../../pulp_python/python-ruby-client.tar "./pulp_python_client-$VERSION.gem" -popd -popd diff --git a/.github/workflows/scripts/post_docs_test.sh b/.github/workflows/scripts/post_docs_test.sh deleted file mode 100644 index 2178e5a84..000000000 --- a/.github/workflows/scripts/post_docs_test.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env sh - -export BASE_ADDR=https://pulp -export CONTENT_ADDR=https://pulp - -pip install -r doc_requirements.txt - -cd docs/_scripts/ -bash ./quickstart.sh diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh deleted file mode 100755 index bd367010a..000000000 --- a/.github/workflows/scripts/publish_client_gem.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -VERSION="$1" - -if [[ -z "$VERSION" ]]; then - echo "No version specified." - exit 1 -fi - -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://rubygems.org/gems/pulp_python_client/versions/$VERSION")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Skipping." - exit -fi - -mkdir -p ~/.gem -touch ~/.gem/credentials -echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials -sudo chmod 600 ~/.gem/credentials -gem push "pulp_python_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh deleted file mode 100755 index 43a31a354..000000000 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -VERSION="$1" - -if [[ -z "$VERSION" ]]; then - echo "No version specified." - exit 1 -fi - -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-python-client/$VERSION/")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Skipping." -else - twine upload -u __token__ -p "$PYPI_API_TOKEN" \ - "dist/pulp_python_client-$VERSION-py3-none-any.whl" \ - "dist/pulp_python-client-$VERSION.tar.gz" -fi diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh deleted file mode 100755 index db944810a..000000000 --- a/.github/workflows/scripts/publish_docs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -mkdir ~/.ssh -touch ~/.ssh/pulp-infra -chmod 600 ~/.ssh/pulp-infra -echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra - -echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts -chmod 644 ~/.ssh/known_hosts - -export PYTHONUNBUFFERED=1 -export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export WORKSPACE=$PWD - -# start the ssh agent -eval "$(ssh-agent -s)" -ssh-add ~/.ssh/pulp-infra - -python3 .github/workflows/scripts/docs-publisher.py --build-type "$1" --branch "$2" - -if [[ "$GITHUB_WORKFLOW" == "Python changelog update" ]]; then - # Do not build bindings docs on changelog update - exit -fi - -mkdir -p ../python-bindings -tar -xvf python-python-client-docs.tar --directory ../python-bindings -pushd ../python-bindings - -# publish to docs.pulpproject.org/pulp_python_client -rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/ - -# publish to docs.pulpproject.org/pulp_python_client/en/{release} -rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/en/"$2" -popd diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh deleted file mode 100755 index 1dd7d883d..000000000 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -VERSION="$1" - -if [[ -z "$VERSION" ]]; then - echo "No version specified." - exit 1 -fi - -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-python/$VERSION/")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_python $VERSION has already been released. Skipping." - exit -fi - -twine upload -u __token__ -p "$PYPI_API_TOKEN" \ -"dist/pulp_python-$VERSION-py3-none-any.whl" \ -"dist/pulp-python-$VERSION.tar.gz" \ -; diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh index 9525f229d..a08353cdb 100755 --- a/.github/workflows/scripts/release.sh +++ b/.github/workflows/scripts/release.sh @@ -10,7 +10,8 @@ then exit 1 fi -NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +# The tail is a necessary workaround to remove the warning from the output. +NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)" echo "Release ${NEW_VERSION}" if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] @@ -20,7 +21,7 @@ then fi towncrier build --yes --version "${NEW_VERSION}" -bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty -bump2version patch --commit +bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump-my-version bump patch --commit git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index b36dd9907..5a4910e30 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -16,10 +16,9 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. source .github/workflows/scripts/utils.sh export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh -export POST_DOCS_TEST=$PWD/.github/workflows/scripts/post_docs_test.sh export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh -# Needed for both starting the service and building the docs. +# Needed for starting the service # Gets set in .github/settings.yml, but doesn't seem to inherited by # this script. export DJANGO_SETTINGS_MODULE=pulpcore.app.settings @@ -27,26 +26,10 @@ export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py export PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" -if [[ "$TEST" = "docs" ]]; then - if [[ "$GITHUB_WORKFLOW" == "Python CI" ]]; then - towncrier build --yes --version 4.0.0.ci - fi - # Unified Docs Build - pulp-docs build - # Legacy Docs Build - cd docs - make PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24PULP_URL" diagrams html - tar -cvf docs.tar ./_build - cd .. - - if [ -f "$POST_DOCS_TEST" ]; then - source "$POST_DOCS_TEST" - fi - exit -fi - REPORTED_STATUS="$(pulp status)" +echo "${REPORTED_STATUS}" + echo "machine pulp login admin password password @@ -57,18 +40,22 @@ cmd_prefix bash -c "chmod 600 ~pulp/.netrc" # Generate bindings ################### +echo "::group::Generate bindings" + touch bindings_requirements.txt pushd ../pulp-openapi-generator # Use app_label to generate api.json and package to produce the proper package name. # Workaround: Domains are not supported by the published bindings. # Sadly: Different pulpcore-versions aren't either... - # So we exclude the prebuilt ones only for domains disabled. - if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ] + # * In the 'pulp' scenario we use the published/prebuilt bindings, so we can test it. + # * In other scenarios we generate new bindings from server spec, so we have a more + # reliable client. + if [ "$TEST" = "pulp" ] then - BUILT_CLIENTS="" - else BUILT_CLIENTS=" python " + else + BUILT_CLIENTS="" fi for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") @@ -79,13 +66,13 @@ pushd ../pulp-openapi-generator # there, because we did not merge plugins into pulpcore back then. MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")" PACKAGE="${MODULE%%.*}" + cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json" if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]] then rm -rf "./${PACKAGE}-client" - cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json" ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" pushd "${PACKAGE}-client" - python setup.py sdist bdist_wheel --python-tag py3 + python -m build popd else if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] @@ -100,11 +87,24 @@ pushd ../pulp-openapi-generator done popd +echo "::endgroup::" + +echo "::group::Debug bindings diffs" + +# Bindings diff for python +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "python-api.json" > "build-api.json" +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "../pulp-openapi-generator/python-api.json" > "test-api.json" +jsondiff --indent 2 build-api.json test-api.json || true +echo "::endgroup::" + # Install test requirements ########################### +# Carry on previous constraints (there might be no such file). +cat *_constraints.txt > bindings_constraints.txt || true +cat .ci/assets/ci_constraints.txt >> bindings_constraints.txt # Add a safeguard to make sure the proper versions of the clients are installed. -echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' >> bindings_constraints.txt cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt @@ -119,7 +119,7 @@ echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations python --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --pyargs pulp_python.tests.unit" +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulp_python.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then @@ -135,16 +135,16 @@ if [ -f "$FUNC_TEST_SCRIPT" ]; then else if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] then - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" else - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel'" - fi + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel'" + fi fi pushd ../pulp-cli pip install -r test_requirements.txt -pytest -v -m "pulp_python" +pytest -v tests -m "pulp_python" popd if [ -f "$POST_SCRIPT" ]; then diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b04cc0f65..fbddb788b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,10 @@ name: "Test" on: workflow_call: + inputs: + matrix_env: + required: true + type: "string" defaults: run: @@ -20,35 +24,38 @@ jobs: strategy: fail-fast: false matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - TEST: lowerbounds + env: "${{ fromJSON(inputs.matrix_env) }}" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - - uses: "actions/download-artifact@v4" + - name: "Download plugin package" + uses: "actions/download-artifact@v8" with: name: "plugin_package" path: "pulp_python/dist/" - - uses: "actions/download-artifact@v4" + - name: "Download API specs" + uses: "actions/download-artifact@v8" + with: + name: "api_spec" + path: "pulp_python/" + + - name: "Download client packages" + uses: "actions/download-artifact@v8" with: name: "python-client.tar" path: "pulp_python" @@ -63,16 +70,14 @@ jobs: - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + pip install build towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - name: "Set environment variables" run: | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - name: "Before Install" + - name: "Prepare Scenario Definition" run: | .github/workflows/scripts/before_install.sh shell: "bash" @@ -81,10 +86,6 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - uses: ruby/setup-ruby@v1 - if: ${{ env.TEST == 'pulp' }} - with: - ruby-version: "2.6" - name: "Install" run: | @@ -96,7 +97,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - name: "Before Script" + - name: "Dump CI Metadata" run: | .github/workflows/scripts/before_script.sh shell: "bash" @@ -105,7 +106,6 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" - name: "Script" run: | @@ -122,22 +122,13 @@ jobs: docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt - name: "Upload Deprecations" - uses: actions/upload-artifact@v4 + uses: "actions/upload-artifact@v5" with: name: "deprecations-${{ matrix.env.TEST }}" path: "pulp_python/deprecations-${{ matrix.env.TEST }}.txt" if-no-files-found: "error" retention-days: 5 overwrite: true - - name: Upload built docs - if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v4 - with: - name: "docs.tar" - path: "pulp_python/docs/docs.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - name: "Logs" if: always() @@ -149,5 +140,5 @@ jobs: docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + docker exec pulp bash -c "pip3 list" || true ... diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index 4565da8cd..39320dcae 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -19,7 +19,7 @@ jobs: update_backport_labels: runs-on: "ubuntu-latest" steps: - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Configure Git with pulpbot name and email" @@ -28,12 +28,11 @@ jobs: git config --global user.email 'pulp-infra@redhat.com' - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install requests pyyaml - echo ::endgroup:: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" - name: "Update labels" run: | python3 .github/workflows/scripts/update_backport_labels.py env: GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" +... diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 2fc9d311c..e6ea0a780 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -12,7 +12,7 @@ on: schedule: # * is a special character in YAML so you have to quote this string # runs at 2:30 UTC every Sunday - - cron: '30 2 * * 0' + - cron: "30 2 * * 0" workflow_dispatch: jobs: @@ -23,27 +23,25 @@ jobs: fail-fast: false steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install gitpython requests packaging jinja2 pyyaml - echo ::endgroup:: + pip install gitpython packaging -r plugin_template/requirements.txt - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -55,22 +53,26 @@ jobs: ../plugin_template/scripts/update_ci.sh - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" + id: "create_pr_main" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_python" committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch main" - body: "" branch: "update-ci/main" base: "main" - commit-message: | - Update CI files - - [noissue] delete-branch: true - - uses: "actions/checkout@v4" + - name: "Mark PR automerge" + working-directory: "pulp_python" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_main.outputs.pull-request-number }}" + if: "steps.create_pr_main.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -79,21 +81,26 @@ jobs: - name: "Run update" working-directory: "pulp_python" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" + id: "create_pr_3_11" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_python" committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 3.11" - body: "" branch: "update-ci/3.11" base: "3.11" - commit-message: | - Update CI files - - [noissue] delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_python" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_3_11.outputs.pull-request-number }}" + if: "steps.create_pr_3_11.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true +... diff --git a/CHANGES.md b/CHANGES.md index 84412881d..465b9afdf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,100 @@ [//]: # (towncrier release notes start) +## 3.12.8 (2025-11-18) {: #3.12.8 } + +No significant changes. + +--- + +## 3.12.7 (2025-07-23) {: #3.12.7 } + +No significant changes. + +--- + +## 3.12.6 (2025-02-26) {: #3.12.6 } + +#### Misc {: #3.12.6-misc } + +- + +--- + +## 3.12.5 (2024-10-25) {: #3.12.5 } + +#### Bugfixes {: #3.12.5-bugfix } + +- Fixed the JSONField specification so it doesn't break ruby bindings. + See context [here](https://github.com/pulp/pulp_rpm/issues/3639). + +--- + +## 3.12.4 (2024-10-14) {: #3.12.4 } + +#### Bugfixes {: #3.12.4-bugfix } + +- Fixed replicate failing on upstream on-demand repositories + [#718](https://github.com/pulp/pulp_python/issues/718) + +--- + +## 3.12.3 (2024-08-26) {: #3.12.3 } + +#### Bugfixes {: #3.12.3-bugfix } + +- Fixed uploads not supporting packages using metadata spec 2.3 + [#682](https://github.com/pulp/pulp_python/issues/682) + +--- + +# ## 3.12.2 (2024-08-21) {: #3.12.2 } + +#### Bugfixes {: #3.12.2-bugfix } + +- Fixed package name normalization issue preventing syncing packages with "." or "_" in their names. + [#716](https://github.com/pulp/pulp_python/issues/716) + +--- + +## 3.12.1 (2024-06-27) {: #3.12.1 } + + +#### Bugfixes {: #3.12.1-bugfix } + +- Fixed the `package_types` filter breaking other remote filters. + [#691](https://github.com/pulp/pulp_python/issues/691) + +--- + +## 3.12.0 (2024-06-25) {: #3.12.0 } + + +#### Features {: #3.12.0-feature } + +- Added RBAC support. + [#399](https://github.com/pulp/pulp_python/issues/399) +- Added Pulp replication support for Python distributions. + [#648](https://github.com/pulp/pulp_python/issues/648) +- Added Domain support. + [#668](https://github.com/pulp/pulp_python/issues/668) + +#### Bugfixes {: #3.12.0-bugfix } + +- Fixed tls_validation not being disabled when set to false on the remote. + [#653](https://github.com/pulp/pulp_python/issues/653) + +#### Deprecations and Removals {: #3.12.0-removal } + +- Raised the minimum `pulpcore` bound to `>=3.49` and dropped support for `python 3.8`. + [#pulpcore](https://github.com/pulp/pulp_python/issues/pulpcore) + +#### Misc {: #3.12.0-misc } + +- + +--- + ## 3.11.1 (2024-04-11) {: #3.11.1 } ### Bugfixes diff --git a/CHANGES/+new-docs-ci.misc b/CHANGES/+new-docs-ci.misc deleted file mode 100644 index 91a157185..000000000 --- a/CHANGES/+new-docs-ci.misc +++ /dev/null @@ -1 +0,0 @@ -Added the Unified Docs CI build-check on PRs. diff --git a/CHANGES/.TEMPLATE.rst b/CHANGES/.TEMPLATE.rst deleted file mode 100644 index 49c2305d7..000000000 --- a/CHANGES/.TEMPLATE.rst +++ /dev/null @@ -1,47 +0,0 @@ -{% if render_title %} -{% if versiondata.name %} -{{ versiondata.name }} {{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.name + versiondata.version + versiondata.date)|length + 4)}} -{% else %} -{{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.version + versiondata.date)|length + 3)}} -{% endif %} -{% endif %} -{% for section, _ in sections.items() %} -{% set underline = underlines[0] %}{% if section %}{{section}} -{{ underline * section|length }}{% set underline = underlines[1] %} - -{% endif %} - -{% if sections[section] %} -{% for category, val in definitions.items() if category in sections[section]%} -{{ definitions[category]['name'] }} -{{ underline * definitions[category]['name']|length }} - -{% if definitions[category]['showcontent'] %} -{% for text, values in sections[section][category].items() %} -- {{ text }} - {{ values|join(',\n ') }} -{% endfor %} - -{% else %} -- {{ sections[section][category]['']|join(', ') }} - -{% endif %} -{% if sections[section][category]|length == 0 %} -No significant changes. - -{% else %} -{% endif %} - -{% endfor %} -{% else %} -No significant changes. - - -{% endif %} -{% endfor %} ----- - - - diff --git a/CHANGES/399.feature b/CHANGES/399.feature deleted file mode 100644 index 732758bb8..000000000 --- a/CHANGES/399.feature +++ /dev/null @@ -1 +0,0 @@ -Added RBAC support. diff --git a/CHANGES/648.feature b/CHANGES/648.feature deleted file mode 100644 index 6c1195673..000000000 --- a/CHANGES/648.feature +++ /dev/null @@ -1 +0,0 @@ -Added Pulp replication support for Python distributions. diff --git a/CHANGES/653.bugfix b/CHANGES/653.bugfix deleted file mode 100644 index 6040d64f2..000000000 --- a/CHANGES/653.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed tls_validation not being disabled when set to false on the remote. diff --git a/CHANGES/668.feature b/CHANGES/668.feature deleted file mode 100644 index b59531ad5..000000000 --- a/CHANGES/668.feature +++ /dev/null @@ -1 +0,0 @@ -Added Domain support. diff --git a/CHANGES/pulpcore.removal b/CHANGES/pulpcore.removal deleted file mode 100644 index 3d9000514..000000000 --- a/CHANGES/pulpcore.removal +++ /dev/null @@ -1 +0,0 @@ -Raised the minimum `pulpcore` bound to `>=3.49` and dropped support for `python 3.8`. diff --git a/MANIFEST.in b/MANIFEST.in index f9759e6f0..e519d8a0a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,3 +8,4 @@ include functest_requirements.txt include test_requirements.txt include unittest_requirements.txt include pulp_python/app/webserver_snippets/* +exclude releasing.md diff --git a/doc_requirements.txt b/doc_requirements.txt index aeca75a05..6456b7d8d 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,16 +1,8 @@ -build -coreapi -django -djangorestframework -django-filter -drf-nested-routers -mistune -plantuml -pyyaml -sphinx -sphinx-rtd-theme -sphinxcontrib-openapi +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template towncrier -twine -# Unified docs pulp-docs @ git+https://github.com/pulp/pulp-docs@main diff --git a/docs/conf.py b/docs/conf.py index 7a8ab8917..53a135f49 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.12.0.dev" +version = "3.12.6.dev" # The full version, including alpha/beta/rc tags. -release = "3.12.0.dev" +release = "3.12.6.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/functest_requirements.txt b/functest_requirements.txt index 8af781c40..d4f65c6ca 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -6,3 +6,4 @@ pytest-custom_exit_code pytest-xdist proxy.py~=2.4.4 trustme~=1.1.0 +pytest-timeout diff --git a/lint_requirements.txt b/lint_requirements.txt index 2422f937b..a29362097 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,7 +5,8 @@ # # For more info visit https://github.com/pulp/plugin_template -# python packages handy for developers, but not required by pulp +bump-my-version check-manifest flake8 +packaging yamllint diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index a74615256..a7f35d1e9 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -10,7 +10,7 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.12.0.dev" + version = "3.12.9.dev" python_package_name = "pulp-python" domain_compatible = True diff --git a/pulp_python/app/fields.py b/pulp_python/app/fields.py new file mode 100644 index 000000000..110d2c55c --- /dev/null +++ b/pulp_python/app/fields.py @@ -0,0 +1,12 @@ +from drf_spectacular.utils import extend_schema_field +from drf_spectacular.types import OpenApiTypes +from rest_framework import serializers + + +@extend_schema_field(OpenApiTypes.OBJECT) +class JSONObjectField(serializers.JSONField): + """A drf JSONField override to force openapi schema to use 'object' type. + + Not strictly correct, but we relied on that for a long time. + See: https://github.com/tfranzel/drf-spectacular/issues/1095 + """ diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index e3e0d2c8a..b17f4cf46 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -17,6 +17,7 @@ from pathlib import PurePath from .utils import ( + canonicalize_name, get_project_metadata_from_artifact, parse_project_metadata, python_content_to_json, @@ -87,6 +88,8 @@ def content_handler(self, path): ).latest("pulp_created") except ObjectDoesNotExist: return None + if len(path.parts) == 2: + path = PurePath(f"simple/{canonicalize_name(path.parts[1])}") rel_path = f"{path}/index.html" try: ca = ( @@ -103,8 +106,9 @@ def content_handler(self, path): return ArtifactResponse(ca.artifact, headers=headers) if name: + normalized = canonicalize_name(name) package_content = PythonPackageContent.objects.filter( - pk__in=self.publication.repository_version.content, name__iexact=name + pk__in=self.publication.repository_version.content, name__normalize=normalized ) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index dc99fc802..301d7a8b0 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -3,6 +3,7 @@ from rest_framework import serializers from pulp_python.app.utils import DIST_EXTENSIONS +from pulp_python.app import fields from pulpcore.plugin.models import Artifact from pulpcore.plugin.util import get_domain from django.db.utils import IntegrityError @@ -28,9 +29,9 @@ class PackageMetadataSerializer(serializers.Serializer): """ last_serial = serializers.IntegerField(help_text=_("Cache value from last PyPI sync")) - info = serializers.JSONField(help_text=_("Core metadata of the package")) - releases = serializers.JSONField(help_text=_("List of all the releases of the package")) - urls = serializers.JSONField() + info = fields.JSONObjectField(help_text=_("Core metadata of the package")) + releases = fields.JSONObjectField(help_text=_("List of all the releases of the package")) + urls = fields.JSONObjectField() class PackageUploadSerializer(serializers.Serializer): diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index f6695ab61..271d76c00 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -327,7 +327,8 @@ def retrieve(self, request, path, meta): elif meta_path.match("*/json"): name = meta_path.parts[0] if name: - package_content = content.filter(name__iexact=name) + normalized = canonicalize_name(name) + package_content = content.filter(name__normalize=normalized) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} if settings.DOMAIN_ENABLED: diff --git a/pulp_python/app/replica.py b/pulp_python/app/replica.py index e59c80543..0a11bfc38 100644 --- a/pulp_python/app/replica.py +++ b/pulp_python/app/replica.py @@ -28,6 +28,14 @@ def remote_extra_fields(self, upstream_distribution): # strain the upstream Pulp. return {"policy": "immediate", "prereleases": True} + def url(self, upstream_distribution): + # Ignore distributions that are only pull-through + repo, pub = upstream_distribution["repository"], upstream_distribution["publication"] + if repo or pub: + return super().url(upstream_distribution) + + return None + def repository_extra_fields(self, remote): # Use autopublish since publications result in faster serving times return {"autopublish": True} diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index ff789b53d..bb588b476 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -8,6 +8,7 @@ from pulpcore.plugin.util import get_domain from pulp_python.app import models as python_models +from pulp_python.app import fields from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata @@ -157,7 +158,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_('A browsable URL for the project and a label for it, separated by a comma.') ) - project_urls = serializers.JSONField( + project_urls = fields.JSONObjectField( required=False, default=dict, help_text=_('A dictionary of labels and URLs for the project.') ) @@ -170,28 +171,28 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ') ) - requires_dist = serializers.JSONField( + requires_dist = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing names of some other distutils project ' 'required by this distribution.') ) - provides_dist = serializers.JSONField( + provides_dist = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing names of a Distutils project which is contained' ' within this distribution.') ) - obsoletes_dist = serializers.JSONField( + obsoletes_dist = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing names of a distutils project\'s distribution which ' 'this distribution renders obsolete, meaning that the two projects should not ' 'be installed at the same time.') ) - requires_external = serializers.JSONField( + requires_external = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing some dependency in the system that the distribution ' 'is to be used.') ) - classifiers = serializers.JSONField( + classifiers = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing classification values for a Python package.') ) diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 3af26a592..a250b9f79 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -83,7 +83,7 @@ def create_bandersnatch_config(remote): config["plugins"]["enabled"] += "prerelease_release\n" if remote.package_types: rrfm = "regex_release_file_metadata" - config["plugins"]["enabled"] += rrfm + config["plugins"]["enabled"] += f"{rrfm}\n" if not config.has_section(rrfm): config.add_section(rrfm) config[rrfm]["any:release_file.packagetype"] = "\n".join(remote.package_types) @@ -256,8 +256,9 @@ async def create_content(self, pkg): for package in dists: entry = parse_metadata(pkg.info, version, package) url = entry.pop("url") + size = package["size"] or None - artifact = Artifact(sha256=entry["sha256"]) + artifact = Artifact(sha256=entry["sha256"], size=size) package = PythonPackageContent(**entry) da = DeclarativeArtifact( diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index f66dcbd7a..126037a89 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -284,19 +284,21 @@ def find_artifact(): if domain: components.insert(2, domain.name) url = "/".join(components) + md5 = artifact.md5 if artifact and artifact.md5 else "" + size = artifact.size if artifact and artifact.size else 0 return { "comment_text": "", - "digests": {"md5": artifact.md5, "sha256": artifact.sha256}, + "digests": {"md5": md5, "sha256": content.sha256}, "downloads": -1, "filename": content.filename, "has_sig": False, - "md5_digest": artifact.md5, + "md5_digest": md5, "packagetype": content.packagetype, "python_version": content.python_version, "requires_python": content.requires_python or None, - "size": artifact.size, - "upload_time": str(artifact.pulp_created), - "upload_time_iso_8601": str(artifact.pulp_created.isoformat()), + "size": size, + "upload_time": str(content.pulp_created), + "upload_time_iso_8601": str(content.pulp_created.isoformat()), "url": url, "yanked": False, "yanked_reason": None diff --git a/pulp_python/tests/functional/api/test_consume_content.py b/pulp_python/tests/functional/api/test_consume_content.py index 9c153cd10..98ac094ee 100644 --- a/pulp_python/tests/functional/api/test_consume_content.py +++ b/pulp_python/tests/functional/api/test_consume_content.py @@ -21,6 +21,7 @@ def test_pip_consume_content( "install", "--no-deps", "--no-cache-dir", + "--no-build-isolation", "--force-reinstall", "--trusted-host", urlsplit(distro.base_url).hostname, diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index ec41a7c8f..5fa9a9825 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -1,6 +1,7 @@ import pytest from urllib.parse import urljoin +from pypi_simple import PyPISimple from pulpcore.tests.functional.utils import PulpTaskError from pulp_python.tests.functional.constants import ( @@ -108,3 +109,29 @@ def test_content_crud( monitor_task(response.task) msg = "The uploaded artifact's sha256 checksum does not match the one provided" assert msg in e.value.task.error["description"] + + +@pytest.mark.parallel +def test_upload_metadata_23_spec(python_content_factory): + """Test that packages using metadata spec 2.3 can be uploaded to pulp.""" + filename = "urllib3-2.2.2-py3-none-any.whl" + with PyPISimple() as client: + page = client.get_project_page("urllib3") + for package in page.packages: + if package.filename == filename: + content = python_content_factory(filename, url=package.url) + assert content.metadata_version == "2.3" + break + + +@pytest.mark.parallel +def test_upload_metadata_24_spec(python_content_factory): + """Test that packages using metadata spec 2.4 can be uploaded to pulp.""" + filename = "urllib3-2.3.0-py3-none-any.whl" + with PyPISimple() as client: + page = client.get_project_page("urllib3") + for package in page.packages: + if package.filename == filename: + content = python_content_factory(filename, url=package.url) + assert content.metadata_version == "2.4" + break diff --git a/pulp_python/tests/functional/api/test_domains.py b/pulp_python/tests/functional/api/test_domains.py index 74316dec3..1efcf1342 100644 --- a/pulp_python/tests/functional/api/test_domains.py +++ b/pulp_python/tests/functional/api/test_domains.py @@ -5,7 +5,12 @@ from pulpcore.app import settings -from pulp_python.tests.functional.constants import PYTHON_URL, PYTHON_EGG_FILENAME +from pulp_python.tests.functional.constants import ( + PYTHON_URL, + PYTHON_EGG_FILENAME, + PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_SM_PACKAGE_COUNT, +) from urllib.parse import urlsplit @@ -47,7 +52,7 @@ def test_domain_object_creation( python_bindings.RepositoriesPythonApi.create(repo_body, pulp_domain=domain.name) assert e.value.status == 400 assert json.loads(e.value.body) == { - "non_field_errors": [f"Objects must all be apart of the {domain_name} domain."] + "non_field_errors": [f"Objects must all be a part of the {domain_name} domain."] } with pytest.raises(python_bindings.ApiException) as e: @@ -55,7 +60,7 @@ def test_domain_object_creation( python_bindings.RepositoriesPythonApi.sync(repo.pulp_href, sync_body) assert e.value.status == 400 assert json.loads(e.value.body) == { - "non_field_errors": [f"Objects must all be apart of the {domain_name} domain."] + "non_field_errors": [f"Objects must all be a part of the {domain_name} domain."] } with pytest.raises(python_bindings.ApiException) as e: @@ -63,7 +68,7 @@ def test_domain_object_creation( python_bindings.PublicationsPypiApi.create(publish_body) assert e.value.status == 400 assert json.loads(e.value.body) == { - "non_field_errors": ["Objects must all be apart of the default domain."] + "non_field_errors": ["Objects must all be a part of the default domain."] } with pytest.raises(python_bindings.ApiException) as e: @@ -73,7 +78,7 @@ def test_domain_object_creation( python_bindings.DistributionsPypiApi.create(distro_body) assert e.value.status == 400 assert json.loads(e.value.body) == { - "non_field_errors": ["Objects must all be apart of the default domain."] + "non_field_errors": ["Objects must all be a part of the default domain."] } @@ -108,7 +113,7 @@ def test_domain_content_upload( python_bindings.ContentPackagesApi.create(**content_body, pulp_domain=domain.name) assert e.value.status == 400 assert json.loads(e.value.body) == { - "non_field_errors": [f"Objects must all be apart of the {domain.name} domain."] + "non_field_errors": [f"Objects must all be a part of the {domain.name} domain."] } # Now create the same content in the second domain @@ -144,7 +149,7 @@ def test_domain_content_replication( python_bindings, python_file, python_repo_factory, - python_publication_factory, + python_remote_factory, python_distribution_factory, monitor_task, monitor_task_group, @@ -154,13 +159,12 @@ def test_domain_content_replication( """Test replication feature through the usage of domains.""" # Set up source domain to replicate from source_domain = domain_factory() - repo = python_repo_factory(pulp_domain=source_domain.name) + repo = python_repo_factory(pulp_domain=source_domain.name, autopublish=True) body = {"relative_path": PYTHON_EGG_FILENAME, "file": python_file, "repository": repo.pulp_href} monitor_task( python_bindings.ContentPackagesApi.create(pulp_domain=source_domain.name, **body).task ) - pub = python_publication_factory(repository=repo, pulp_domain=source_domain.name) - python_distribution_factory(publication=pub.pulp_href, pulp_domain=source_domain.name) + python_distribution_factory(repository=repo, pulp_domain=source_domain.name) # Create the replica domain replica_domain = domain_factory() @@ -171,6 +175,7 @@ def test_domain_content_replication( "domain": source_domain.name, "username": bindings_cfg.username, "password": bindings_cfg.password, + "tls_validation": False, } upstream_pulp = gen_object_with_cleanup( pulpcore_bindings.UpstreamPulpsApi, upstream_pulp_body, pulp_domain=replica_domain.name @@ -194,6 +199,25 @@ def test_domain_content_replication( assert all(1 == x for x in counts.values()), f"Replica had more than 1 object {counts}" + # Test that we can replicate from an Upstream on-demand source (syncs are on-demand by default) + remote = python_remote_factory( + includes=PYTHON_SM_PROJECT_SPECIFIER, pulp_domain=source_domain.name + ) + body = {"remote": remote.pulp_href} + monitor_task(python_bindings.RepositoriesPythonApi.sync(repo.pulp_href, body).task) + + response = pulpcore_bindings.UpstreamPulpsApi.replicate(upstream_pulp.pulp_href) + monitor_task_group(response.task_group) + + response = python_bindings.ContentPackagesApi.list(pulp_domain=replica_domain.name) + assert PYTHON_SM_PACKAGE_COUNT + 1 == response.count + response = python_bindings.PublicationsPypiApi.list(pulp_domain=replica_domain.name) + assert 2 == response.count + add_to_cleanup(python_bindings.PublicationsPypiApi, response.results[0]) + assert 1 == python_bindings.RepositoriesPythonApi.list(pulp_domain=replica_domain.name).count + assert 1 == python_bindings.DistributionsPypiApi.list(pulp_domain=replica_domain.name).count + assert 1 == python_bindings.RemotesPythonApi.list(pulp_domain=replica_domain.name).count + @pytest.fixture def shelf_reader_cleanup(): @@ -253,6 +277,7 @@ def test_domain_pypi_apis( "pip", "install", "--no-deps", + "--no-build-isolation", "--trusted-host", urlsplit(distro.base_url).hostname, "-i", diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 8c58fb816..11ca94494 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -87,3 +87,32 @@ def test_full_pulp_to_pulp_sync( repo3 = python_repo_with_sync(remote3) summary2 = python_content_summary(repository_version=repo3.latest_version_href) assert summary2.present["python.python"]["count"] == PYTHON_MD_PACKAGE_COUNT + + +@pytest.mark.parallel +def test_pulp2pulp_sync_with_oddities( + python_repo_with_sync, + python_remote_factory, + python_publication_factory, + python_distribution_factory, + python_content_summary, +): + """Test that Pulp can handle syncing packages with wierd names.""" + remote = python_remote_factory(includes=["oslo.utils"], url="iframe.php?url=https%3A%2F%2Fpypi.org") + repo = python_repo_with_sync(remote) + distro = python_distribution_factory(repository=repo) + summary = python_content_summary(repository_version=repo.latest_version_href) + # Test pulp 2 pulp full sync w/ live pypi apis + remote2 = python_remote_factory(includes=[], url=distro.base_url) + repo2 = python_repo_with_sync(remote2) + summary2 = python_content_summary(repository_version=repo2.latest_version_href) + assert summary2.present["python.python"]["count"] > 0 + assert summary.present["python.python"]["count"] == summary2.present["python.python"]["count"] + # Test w/ publication + pub = python_publication_factory(repository=repo) + distro2 = python_distribution_factory(publication=pub) + remote3 = python_remote_factory(includes=[], url=distro2.base_url) + repo3 = python_repo_with_sync(remote3) + summary3 = python_content_summary(repository_version=repo3.latest_version_href) + assert summary3.present["python.python"]["count"] > 0 + assert summary.present["python.python"]["count"] == summary3.present["python.python"]["count"] diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 1e557f367..034f8f83f 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -224,26 +224,6 @@ def test_twine_upload( check=True, ) - # Test re-uploading same packages with --skip-existing works - output = subprocess.run( - ( - "twine", - "upload", - "--repository-url", - url, - dist_dir / "*", - "-u", - username, - "-p", - password, - "--skip-existing", - ), - capture_output=True, - check=True, - text=True - ) - assert output.stdout.count("Skipping") == 2 - @pytest.mark.parallel def test_simple_redirect_with_publications( diff --git a/pulp_python/tests/functional/api/test_sync.py b/pulp_python/tests/functional/api/test_sync.py index 20e6b19c9..0a846cf61 100644 --- a/pulp_python/tests/functional/api/test_sync.py +++ b/pulp_python/tests/functional/api/test_sync.py @@ -264,6 +264,23 @@ def test_sync_platform_exclude( assert summary.present["python.python"]["count"] == SCIPY_COUNTS["no_os"] +@pytest.mark.parallel +def test_sync_multiple_filters( + python_repo_with_sync, python_remote_factory, python_content_summary +): + """Tests sync with multiple filters.""" + remote = python_remote_factory( + includes=PYTHON_LG_PROJECT_SPECIFIER, + package_types=["bdist_wheel"], + keep_latest_packages=1, + prereleases=False + ) + repo = python_repo_with_sync(remote) + + summary = python_content_summary(repository_version=repo.latest_version_href) + assert summary.present["python.python"]["count"] == PYTHON_LG_FIXTURE_COUNTS["multi"] + + @pytest.mark.parallel def test_proxy_sync( python_bindings, diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index e449fe92e..231ae1bcf 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -124,17 +124,22 @@ "aiohttp", # matches 7 "bcrypt", # matches 8 "celery", # matches 13 + "crane", # matches 0 "Django", # matches 31 + "pulp-2to3-migration", # matches 2 "pytz", # matches 6 "scipy", # matches 23 + "setuptools", # matches 2 "shelf-reader", # matches 2 + "twine", # matches 14 ] -PYTHON_LG_PACKAGE_COUNT = 90 +PYTHON_LG_PACKAGE_COUNT = 108 PYTHON_LG_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_LG_PACKAGE_COUNT} PYTHON_LG_FIXTURE_COUNTS = { - "latest_3": 49, - "sdist": 27, - "bdist_wheel": 63, + "latest_3": 59, + "sdist": 36, + "bdist_wheel": 72, + "multi": 36, # keep_latest=1, package_types="bdist_wheel", prereleases=False } DJANGO_LATEST_3 = 4 # latest version has 2 dists, each other has 1 diff --git a/pyproject.toml b/pyproject.toml index 3bbd14f6f..4aa2b5cff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,4 +27,51 @@ ignore = [ ".ci/**", "lint_requirements.txt", ".flake8", + "AGENTS.md", + "CLAUDE.md", ] + +[tool.bumpversion] +# This section is managed by the plugin template. Do not edit manually. + +current_version = "3.12.9.dev" +commit = false +tag = false +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" +serialize = [ + "{major}.{minor}.{patch}.{release}", + "{major}.{minor}.{patch}", + "{major}.{minor}.{alpha}{patch}.{release}", + "{major}.{minor}.{alpha}{patch}", +] + +[tool.bumpversion.parts.alpha] +# This section is managed by the plugin template. Do not edit manually. + +# This is sort of a hack. In PEP440 prerelease markers work quite differently. +# But this fits best with the way we have been doing release versions. +optional_value = "final" +values = [ + "0a", + "final", +] +independent = true + +[tool.bumpversion.parts.release] +# This section is managed by the plugin template. Do not edit manually. + +optional_value = "prod" +values = [ + "dev", + "prod", +] + +[[tool.bumpversion.files]] +# This section is managed by the plugin template. Do not edit manually. + +filename = "./pulp_python/app/__init__.py" +search = "version = \"{current_version}\"" +replace = "version = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "./setup.py" diff --git a/releasing.md b/releasing.md new file mode 100644 index 000000000..db7470b28 --- /dev/null +++ b/releasing.md @@ -0,0 +1,29 @@ +[//]: # "WARNING: DO NOT EDIT!" +[//]: # "" +[//]: # "This file was generated by plugin_template, and is managed by it. Please use" +[//]: # "'./plugin-template --github pulp_python' to update this file." +[//]: # "" +[//]: # "For more info visit https://github.com/pulp/plugin_template" +# Releasing (For Internal Use) + +This document outlines the steps to perform a release. + +### Determine if a Release is Required +- Make sure to have GitPython python package installed +- Run the release checker script: + ``` + python3 .ci/scripts/check_release.py + ``` + +### Release a New Y-version (e.g., 3.23.0) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_python/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. +- Look for the "Bump minor version" pull request and merge it. +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_python/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. + +### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_python/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. + +## Final Steps +- Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulp_python/actions/workflows/publish.yml) workflow succeeds). +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). +- [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). diff --git a/requirements.txt b/requirements.txt index 91bc710c4..362109804 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ pulpcore>=3.49.0,<3.70.0 -pkginfo>=1.8.2,<1.9.7 +pkginfo>=1.12.0,<1.13.0 bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 diff --git a/setup.py b/setup.py index b66e4d3cb..cf656c71c 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.12.0.dev", + version="3.12.9.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", diff --git a/template_config.yml b/template_config.yml index 39538a37b..84ae8e666 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,70 +1,95 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-326-ge5addc7 +# generated with plugin_template +# +# After editing this file please always reapply the plugin template before committing any changes. -api_root: /pulp/ +--- +api_root: "/pulp/" black: false check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true -ci_base_image: ghcr.io/pulp/pulp-ci-centos9 +ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" ci_env: {} -ci_trigger: '{pull_request: {branches: [''*'']}}' -ci_update_docs: false -cli_package: pulp-cli -cli_repo: https://github.com/pulp/pulp-cli.git +ci_trigger: "{pull_request: {branches: ['*']}}" +cli_package: "pulp-cli" +cli_repo: "https://github.com/pulp/pulp-cli.git" core_import_allowed: [] deploy_client_to_pypi: true deploy_client_to_rubygems: true deploy_to_pypi: true disabled_redis_runners: [] -doc_requirements_from_pulpcore: true docker_fixtures: false -docs_test: true -extra_docs_requirements: [] +extra_files: [] flake8: true flake8_ignore: [] -github_org: pulp -issue_tracker: github -kanban: true +github_org: "pulp" latest_release_branch: null lint_requirements: true -noissue_marker: '[noissue]' +os_required_packages: [] parallel_test_workers: 8 -plugin_app_label: python -plugin_default_branch: main -plugin_name: pulp_python +plugin_app_label: "python" +plugin_default_branch: "main" +plugin_name: "pulp_python" plugins: -- app_label: python - name: pulp_python -post_job_template: null -pre_job_template: null -publish_docs_to_pulpprojectdotorg: true + - app_label: "python" + name: "pulp_python" pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} pulp_env_s3: {} -pulp_scheme: https +pulp_scheme: "https" pulp_settings: - allowed_export_paths: /tmp - allowed_import_paths: /tmp + allowed_export_paths: "/tmp" + allowed_import_paths: "/tmp" orphan_protection_time: 0 - pypi_api_hostname: https://pulp:443 + pypi_api_hostname: "https://pulp:443" pulp_settings_azure: + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.azure_storage.AzureStorage" + OPTIONS: + account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + account_name: "devstoreaccount1" + azure_container: "pulp-test" + connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;" + expiration_secs: 120 + location: "pulp3" + overwrite_files: true + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" domain_enabled: true pulp_settings_gcp: null pulp_settings_s3: + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.s3boto3.S3Boto3Storage" + OPTIONS: + access_key: "AKIAIT2Z5TDYPX3ARJBA" + addressing_style: "path" + bucket_name: "pulp3" + default_acl: "@none" + endpoint_url: "http://minio:9000" + region_name: "eu-central-1" + secret_key: "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS" + signature_version: "s3v4" + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" domain_enabled: true pydocstyle: true -release_email: pulp-infra@redhat.com -release_user: pulpbot +release_email: "pulp-infra@redhat.com" +release_user: "pulpbot" stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 stalebot_limit_to_pulls: true -supported_release_branches: ['3.11'] +supported_release_branches: + - "3.11" sync_ci: true test_azure: true test_cli: true @@ -72,9 +97,7 @@ test_deprecations: true test_gcp: false test_lowerbounds: true test_performance: false -test_reroute: true test_s3: true use_issue_template: true -use_legacy_docs: true -use_unified_docs: true +...