diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index c4084695a..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[bumpversion] -current_version = 3.11.0.dev -commit = False -tag = False -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? -serialize = - {major}.{minor}.{patch}.{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = prod -first_value = dev -values = - dev - prod - -[bumpversion:file:./pulp_python/app/__init__.py] - -[bumpversion:file:./setup.py] - -[bumpversion:file:./docs/conf.py] diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 0eb5d5ccb..fb9ed4a1c 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,33 +1,29 @@ -FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }} +FROM {{ image.ci_base }} +{%- if image.webserver_snippet %} -# Add source directories to container -{% for item in plugins %} -{% if item.source.startswith("./") or item.ci_requirements | default(false) %} -ADD ./{{ item.name }} ./{{ item.name }} -{% endif %} -{% endfor %} +ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf +{%- endif %} -# Install python packages +{%- for item in extra_files | default([]) %} -RUN pip3 install -{%- for item in plugins -%} -{%- if item.name == "pulp-certguard" -%} -{{ " " }}python-dateutil rhsm -{%- endif -%} -{{ " " }}{{ item.source }} -{%- if item.name == "pulpcore" -%} -{%- if s3_test | default(false) -%} -[s3] -{%- elif azure_test | default(false) -%} -[azure] -{%- elif gcp_test | default(false) -%} -[google] +ADD ./{{ item.origin }} {{ item.destination }} +{%- endfor %} + +# This MUST be the ONLY call to pip install in inside the container. +RUN pip3 install --upgrade pip setuptools wheel && \ + rm -rf /root/.cache/pip && \ + pip3 install {{ image.source }} +{%- if image.upperbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt {%- endif -%} +{%- if image.lowerbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt {%- endif -%} -{%- if item.ci_requirements | default(false) -%} -{{ " " }}-r ./{{ item.name }}/ci_requirements.txt +{%- if image.ci_requirements | default(false) -%} +{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt {%- endif -%} -{%- endfor %} +{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \ + rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} @@ -44,11 +40,8 @@ ENV {{ key | upper }}={{ value }} USER pulp:pulp RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link -USER root:root -{% for item in plugins %} -RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ - ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true -{% endfor %} +RUN mkdir /var/lib/pulp/.config +USER root:root ENTRYPOINT ["/init"] diff --git a/.ci/ansible/build_container.yaml b/.ci/ansible/build_container.yaml index c380b430a..0a188cba9 100644 --- a/.ci/ansible/build_container.yaml +++ b/.ci/ansible/build_container.yaml @@ -1,15 +1,14 @@ # Ansible playbook to create the pulp service containers image --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Generate Containerfile from template" - template: - src: Containerfile.j2 - dest: Containerfile - + ansible.builtin.template: + src: "Containerfile.j2" + dest: "Containerfile" - name: "Build pulp image" # We build from the ../.. (parent dir of pulpcore git repo) Docker build # "context" so that repos like pulp-smash are accessible to Docker diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 024df4e4c..dfe2851de 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -26,35 +26,3 @@ API_ROOT = {{ api_root | repr }} {% endfor %} {% endif %} -{% if s3_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -MEDIA_ROOT = "" -AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" -AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" -AWS_S3_REGION_NAME = "eu-central-1" -AWS_S3_ADDRESSING_STYLE = "path" -S3_USE_SIGV4 = True -AWS_S3_SIGNATURE_VERSION = "s3v4" -AWS_STORAGE_BUCKET_NAME = "pulp3" -AWS_S3_ENDPOINT_URL = "http://minio:9000" -AWS_DEFAULT_ACL = "@none None" -{% endif %} - -{% if azure_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" -MEDIA_ROOT = "" -AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -AZURE_ACCOUNT_NAME = "devstoreaccount1" -AZURE_CONTAINER = "pulp-test" -AZURE_LOCATION = "pulp3" -AZURE_OVERWRITE_FILES = True -AZURE_URL_EXPIRATION_SECS = 120 -AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;' -{% endif %} - -{% if gcp_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" -MEDIA_ROOT = "" -GS_BUCKET_NAME = "gcppulp" -GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443" -{% endif %} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index b19a2054f..acdc22ad3 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -1,33 +1,26 @@ # Ansible playbook to start the pulp service container and its supporting services --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Create Settings Directories" - file: + ansible.builtin.file: path: "{{ item }}" - state: directory + state: "directory" mode: "0755" loop: - - settings - - ssh - - ~/.config/pulp_smash + - "settings" - name: "Generate Pulp Settings" template: - src: settings.py.j2 - dest: settings/settings.py - - - name: "Configure pulp-smash" - copy: - src: smash-config.json - dest: ~/.config/pulp_smash/settings.json + src: "settings.py.j2" + dest: "settings/settings.py" - name: "Setup docker networking" docker_network: - name: pulp_ci_bridge + name: "pulp_ci_bridge" - name: "Start Service Containers" docker_container: @@ -37,24 +30,24 @@ recreate: true privileged: true networks: - - name: pulp_ci_bridge + - name: "pulp_ci_bridge" aliases: "{{ item.name }}" volumes: "{{ item.volumes | default(omit) }}" env: "{{ item.env | default(omit) }}" command: "{{ item.command | default(omit) }}" - state: started + state: "started" loop: "{{ services | default([]) }}" - name: "Retrieve Docker Network Info" docker_network_info: - name: pulp_ci_bridge - register: pulp_ci_bridge_info + name: "pulp_ci_bridge" + register: "pulp_ci_bridge_info" - name: "Update /etc/hosts" lineinfile: - path: /etc/hosts + path: "/etc/hosts" regexp: "\\s{{ item.value.Name }}\\s*$" - line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}" + line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}" loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}" become: true @@ -63,19 +56,19 @@ aws_access_key: "{{ minio_access_key }}" aws_secret_key: "{{ minio_secret_key }}" s3_url: "http://minio:9000" - region: eu-central-1 - name: pulp3 - state: present - when: s3_test | default(false) + region: "eu-central-1" + name: "pulp3" + state: "present" + when: "s3_test | default(false)" - block: - name: "Wait for Pulp" uri: url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/" - follow_redirects: all - validate_certs: no - register: result - until: result.status == 200 + follow_redirects: "all" + validate_certs: "no" + register: "result" + until: "result.status == 200" retries: 12 delay: 5 rescue: @@ -83,22 +76,14 @@ command: "docker logs pulp" failed_when: true - - block: - - name: "Check version of component being tested" - assert: - that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver) - fail_msg: | - Component {{ component_name }} was expected to be installed in version {{ component_version }}. - Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}. - rescue: - - name: "Check version of component being tested (legacy)" - assert: - that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] | canonical_semver == (component_version | canonical_semver) - fail_msg: | - Component {{ legacy_component_name }} was expected to be installed in version {{ component_version }}. - Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] }}. + - name: "Check version of component being tested" + assert: + that: + - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)" + fail_msg: | + Component {{ item.app_label }} was expected to be installed in version {{ component_version }}. + Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}. + loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}" - name: "Set pulp password in .netrc" copy: @@ -108,9 +93,20 @@ login admin password password -- hosts: pulp +- hosts: "pulp" gather_facts: false tasks: + - name: "Create directory for pulp-smash config" + ansible.builtin.file: + path: "/var/lib/pulp/.config/pulp_smash/" + state: "directory" + mode: "0755" + + - name: "Configure pulp-smash" + ansible.builtin.copy: + src: "smash-config.json" + dest: "/var/lib/pulp/.config/pulp_smash/settings.json" + - name: "Set pulp admin password" command: cmd: "pulpcore-manager reset-admin-password --password password" diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt new file mode 100644 index 000000000..c9198f19a --- /dev/null +++ b/.ci/assets/ci_constraints.txt @@ -0,0 +1,19 @@ +# Pulpcore versions without the openapi command do no longer work in the CI +# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70 +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.* + + +tablib!=3.6.0 +# 3.6.0: This release introduced a regression removing the "html" optional dependency. + + +multidict!=6.3.0 +# This release failed the lower bounds test for some case sensitivity in CIMultiDict. + + +azure-storage-blob!=12.28.* +# Apparently does not work with current azurite. + + +pycares<5 +# older aiodns versions don't pin pycares UB, and are broken by pycares>=5 diff --git a/.ci/assets/release_requirements.txt b/.ci/assets/release_requirements.txt index c064e9477..6635a8723 100644 --- a/.ci/assets/release_requirements.txt +++ b/.ci/assets/release_requirements.txt @@ -1,3 +1,3 @@ -bump2version +bump-my-version gitpython towncrier diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py new file mode 100755 index 000000000..66c494e97 --- /dev/null +++ b/.ci/scripts/calc_constraints.py @@ -0,0 +1,142 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import fileinput +import urllib.request +import sys +from packaging.requirements import Requirement +from packaging.version import Version +import yaml + +try: + import tomllib +except ImportError: + import tomli as tomllib + + +CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" + + +def fetch_pulpcore_upper_bound(requirement): + with urllib.request.urlopen(CORE_TEMPLATE_URL) as f: + template = yaml.safe_load(f.read()) + supported_versions = template["supported_release_branches"] + supported_versions.append(template["latest_release_branch"]) + applicable_versions = sorted( + requirement.specifier.filter((Version(v) for v in supported_versions)) + ) + if len(applicable_versions) == 0: + raise Exception("No supported pulpcore version in required range.") + return f"{requirement.name}~={applicable_versions[-1]}" + + +def split_comment(line): + split_line = line.split("#", maxsplit=1) + try: + comment = " # " + split_line[1].strip() + except IndexError: + comment = "" + return split_line[0].strip(), comment + + +def to_upper_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + if requirement.name == "pulpcore": + # An exception to allow for pulpcore deprecation policy. + return fetch_pulpcore_upper_bound(requirement) + # skip requirement with environment scopes. E.g 'foo==1.0.0;python_version>=3.9' + if requirement.marker: + return f"# ENVIRONMENT IS UNTRACKABLE: {req}" + for spec in requirement.specifier: + if spec.operator == "~=": + return f"# NO BETTER CONSTRAINT: {req}" + if spec.operator == "<=": + operator = "==" + max_version = spec.version + return f"{requirement.name}{operator}{max_version}" + if spec.operator == "<": + operator = "~=" + version = Version(spec.version) + if version.micro != 0: + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" + elif version.minor != 0: + max_version = f"{version.major}.{version.minor - 1}" + elif version.major != 0: + max_version = f"{version.major - 1}.0" + else: + return f"# NO BETTER CONSTRAINT: {req}" + return f"{requirement.name}{operator}{max_version}" + return f"# NO UPPER BOUND: {req}" + + +def to_lower_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + for spec in requirement.specifier: + if spec.operator == ">=": + min_version = spec.version + if requirement.name == "pulpcore": + # Currently an exception to allow for pulpcore bugfix releases. + # TODO Semver libraries should be allowed too. + operator = "~=" + if len(Version(min_version).release) != 3: + raise RuntimeError("Pulpcore lower bound must be in the form '>=x.y.z'.") + else: + operator = "==" + return f"{requirement.name}{operator}{min_version}" + return f"# NO LOWER BOUND: {req}" + + +def main(): + """Calculate constraints for the lower bound of dependencies where possible.""" + parser = argparse.ArgumentParser( + prog=sys.argv[0], + description="Calculate constraints for the lower or upper bound of dependencies where " + "possible.", + ) + parser.add_argument("-u", "--upper", action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true") + parser.add_argument("filename", nargs="*") + args = parser.parse_args() + + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) + else: + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/calc_deps_lowerbounds.py b/.ci/scripts/calc_deps_lowerbounds.py deleted file mode 100755 index 2bb48e250..000000000 --- a/.ci/scripts/calc_deps_lowerbounds.py +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -from packaging.requirements import Requirement - - -def main(): - """Calculate the lower bound of dependencies where possible.""" - with open("requirements.txt") as req_file: - for line in req_file: - try: - requirement = Requirement(line) - except ValueError: - print(line.strip()) - else: - for spec in requirement.specifier: - if spec.operator == ">=": - if requirement.name == "pulpcore": - operator = "~=" - else: - operator = "==" - min_version = str(spec)[2:] - print(f"{requirement.name}{operator}{min_version}") - break - else: - print(line.strip()) - - -if __name__ == "__main__": - main() diff --git a/.ci/scripts/changelog.py b/.ci/scripts/changelog.py deleted file mode 100755 index 3f061fada..000000000 --- a/.ci/scripts/changelog.py +++ /dev/null @@ -1,70 +0,0 @@ -import re -import os -import requests -from packaging.version import Version -from git import Repo - -repo = Repo(os.getcwd()) -heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp_python.git").split("\n") -branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])] -branches.sort(key=lambda ver: Version(ver), reverse=True) - - -def get_changelog(branch): - """ - Get changelog file for a given branch. - - """ - return requests.get( - f"https://raw.githubusercontent.com/pulp/pulp_python/{branch}/CHANGES.rst" - ).text - - -def get_changelog_releases(changelog): - """ - Get all versions in changelog. - - """ - versions = re.findall( - r"([0-9]+)\.([0-9]+)\.([0-9][0-9ab]*) \([0-9]{4}-[0-9]{2}-[0-9]{2}\)", changelog - ) - return {".".join(v) for v in versions} - - -def get_changelog_entry(changelog, version): - """ - Get changelog entry for a given version. - - """ - entries = changelog.split(f"{version} (")[1].split("=====\n") - header = f"{version} ({entries[0]}=====\n" - text = "\n\n\n".join(entries[1].split("\n\n\n")[0:-1]) - return header + text + "\n\n\n" - - -main_changelog = get_changelog("main") -main_entries = get_changelog_releases(main_changelog) -entries_list = list(main_entries) -to_add = {} -for branch in branches: - changelog = get_changelog(branch) - entries = get_changelog_releases(changelog) - for entry in entries.difference(main_entries): - description = get_changelog_entry(changelog, entry) - entries_list.append(entry) - print(description) - to_add[entry] = description - -entries_list.sort(key=lambda ver: Version(ver), reverse=True) -for version in sorted(to_add, key=lambda ver: Version(ver)): - next_version = entries_list[entries_list.index(version) + 1] - new_changelog = main_changelog.split(f"{next_version} (")[0] + to_add[version] - new_changelog = new_changelog + f"{next_version} (" - new_changelog = new_changelog + main_changelog.split(f"{next_version} (")[1] - main_changelog = new_changelog - -with open("CHANGES.rst", "w") as f: - f.write(main_changelog) - -if to_add: - repo.git.commit("-m", "Update Changelog\n\n[noissue]", "CHANGES.rst") diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 70b48a2a6..611c882ca 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -1,112 +1,191 @@ #!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# "pyyaml>=6.0.3,<6.1.0", +# ] +# /// import argparse import re import os +import sys +import tomllib +import typing as t +from pathlib import Path + import yaml -from tempfile import TemporaryDirectory from packaging.version import Version from git import Repo -UPSTREAM_REMOTE = "https://github.com/pulp/pulp_python.git" -DEFAULT_BRANCH = "main" RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" -Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"] -Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"] +Y_CHANGELOG_EXTS = [".feature"] +Z_CHANGELOG_EXTS = [".bugfix", ".misc"] -def main(): +def options() -> argparse.Namespace: """Check which branches need a release.""" parser = argparse.ArgumentParser() parser.add_argument( "--branches", default="supported", help="A comma separated list of branches to check for releases. Can also use keyword: " - "'supported'. Defaults to 'supported', see `ci_update_branches` in " + "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) - opts = parser.parse_args() - - with TemporaryDirectory() as d: - # Clone from upstream to ensure we have updated branches & main - repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none") - heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")] - available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] - available_branches.sort(key=lambda ver: Version(ver)) - available_branches.append(DEFAULT_BRANCH) - - branches = opts.branches - if branches == "supported": - with open(f"{d}/template_config.yml", mode="r") as f: - tc = yaml.safe_load(f) - branches = tc["ci_update_branches"] - branches.append(DEFAULT_BRANCH) + parser.add_argument( + "--no-fetch", + default=False, + action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true", + help="Don't fetch remote. Run faster at the expense of maybe being outdated.", + ) + return parser.parse_args() + + +def template_config() -> dict[str, t.Any]: + # Assume this script lies in .ci/scripts + path = Path(__file__).absolute().parent.parent.parent / "template_config.yml" + return yaml.safe_load(path.read_text()) + + +def current_version(repo: Repo, commitish: str) -> Version: + try: + pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml")) + try: + current_version = pyproject_toml["project"]["version"] + except Exception: + current_version = pyproject_toml["tool"]["bumpversion"]["current_version"] + except Exception: + current_version = repo.git.grep( + "current_version", commitish, "--", ".bumpversion.cfg" + ).split("=")[-1] + return Version(current_version) + + +def check_pyproject_dependencies(repo: Repo, from_commit: str, to_commit: str) -> list[str]: + try: + new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml")) + try: + new_dependencies = set(new_pyproject["project"]["dependencies"]) + except KeyError: + # New branch does not declare dependencies in pyproject.toml. + # Assume no release needed for this reason. + return [] + old_pyproject = tomllib.loads(repo.git.show(f"{from_commit}:pyproject.toml")) + old_dependencies = set(old_pyproject["project"]["dependencies"]) + if old_dependencies != new_dependencies: + return ["dependencies"] else: - branches = branches.split(",") - - if diff := set(branches) - set(available_branches): - print(f"Supplied branches contains non-existent branches! {diff}") - exit(1) - - print(f"Checking for releases on branches: {branches}") - - releases = [] - for branch in branches: - if branch != DEFAULT_BRANCH: - # Check if a Z release is needed - changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") - z_release = False - for change in changes.split("\n"): - # Check each changelog file to make sure everything checks out - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - print( - f"Warning: A non-backported changelog ({change}) is present in the " - f"{branch} release branch!" - ) - elif ext in Z_CHANGELOG_EXTS: - z_release = True - if z_release: - # Blobless clone does not have file contents for Z branches, - # check commit message for last Z bump - git_branch = f"origin/{branch}" - next_version = repo.git.log( - "--oneline", "--grep=Bump to", "-n 1", git_branch, "--", ".bumpversion.cfg" - ).split("to")[-1] - next_version = Version(next_version) + return [] + except Exception as e: + print(f"WARNING: Comparing the dependencies in pyproject.toml failed. ({e})") + # Gathering more details failed. + return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."] + + +def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int: + DEFAULT_BRANCH: str = template_config["plugin_default_branch"] + + repo = Repo() + + upstream_default_branch = next( + (branch for branch in repo.branches if branch.name == DEFAULT_BRANCH) + ).tracking_branch() + remote = upstream_default_branch.remote_name + if not options.no_fetch: + repo.remote(remote).fetch() + + # Warning: This will not work if branch names contain "/" but we don't really care here. + heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")] + available_branches = [h for h in heads if re.fullmatch(RELEASE_BRANCH_REGEX, h)] + available_branches.sort(key=lambda ver: Version(ver)) + available_branches.append(DEFAULT_BRANCH) + + branches = options.branches + if branches == "supported": + tc = yaml.safe_load(repo.git.show(f"{upstream_default_branch}:template_config.yml")) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) + else: + branches = set(branches.split(",")) + + if diff := branches - set(available_branches): + print(f"Supplied branches contains non-existent branches! {diff}") + return 1 + + branches = [branch for branch in available_branches if branch in branches] + branches.reverse() + + print(f"Checking for releases on branches: {branches}") + + releases = [] + for branch in branches: + if branch != DEFAULT_BRANCH: + # Check if a Z release is needed + reasons = [] + changes = repo.git.ls_tree("-r", "--name-only", f"{remote}/{branch}", "CHANGES/") + z_changelog = False + for change in changes.split("\n"): + # Check each changelog file to make sure everything checks out + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: print( - f"A Z-release is needed for {branch}, " - f"New Version: {next_version.base_version}" + f"Warning: A non-backported changelog ({change}) is present in the " + f"{branch} release branch!" ) + elif ext in Z_CHANGELOG_EXTS: + z_changelog = True + if z_changelog: + reasons.append("Backports") + + last_tag = repo.git.describe("--tags", "--abbrev=0", f"{remote}/{branch}") + req_txt_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "requirements.txt" + ) + if req_txt_diff: + reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.extend(check_pyproject_dependencies(repo, last_tag, f"{remote}/{branch}")) + + if reasons: + curr_version = Version(last_tag) + assert curr_version.base_version.startswith( + branch + ), "Current-version has to belong to the current branch!" + next_version = Version(f"{branch}.{curr_version.micro + 1}") + print( + f"A Z-release is needed for {branch}, " + f"Prev: {last_tag}, " + f"Next: {next_version.base_version}, " + f"Reason: {','.join(reasons)}" + ) + releases.append(next_version) + else: + # Check if a Y release is needed + changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") + for change in changes.split("\n"): + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: + # We don't put Y release bumps in the commit message, check file instead. + # The 'current_version' is always the dev of the next version to release. + next_version = current_version(repo, DEFAULT_BRANCH).base_version + print(f"A new Y-release is needed! New Version: {next_version}") releases.append(next_version) - else: - # Check if a Y release is needed - changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") - for change in changes.split("\n"): - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - # We don't put Y release bumps in the commit message, check file instead - # The 'current_version' is always the next version to release - next_version = repo.git.grep( - "current_version", DEFAULT_BRANCH, "--", ".bumpversion.cfg" - ).split("=")[-1] - next_version = Version(next_version) - print( - f"A new Y-release is needed! New Version: {next_version.base_version}" - ) - releases.append(next_version) - break - - if len(releases) == 0: - print("No new releases to perform.") + break + + if len(releases) == 0: + print("No new releases to perform.") + + return 0 if __name__ == "__main__": - main() + sys.exit(main(options(), template_config())) diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index 671bade19..cf9efbe97 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,11 +5,13 @@ # # For more info visit https://github.com/pulp/plugin_template +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -20,17 +22,33 @@ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -49,18 +67,21 @@ and req.name != "pulp-python-client" ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") - ops = [op for op, ver in req.specs] - spec = str(req.specs) + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) +if __name__ == "__main__": + main() diff --git a/.ci/scripts/clean_gh_release_notes.py b/.ci/scripts/clean_gh_release_notes.py new file mode 100755 index 000000000..2bf5f4c27 --- /dev/null +++ b/.ci/scripts/clean_gh_release_notes.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# This script is running with elevated privileges from the main branch against pull requests. +# +# It cleans the input from artifacts which are used by the pulp documentation internally, +# but clutter for GitHub releases + +import sys + +NOTE = """ +> [!NOTE] +> Official changes are available on [Pulp docs]({docs_url})\ +""" + + +def main(): + plugin_name = sys.argv[1] + version_str = sys.argv[2] + docs_url = f"https://pulpproject.org/{plugin_name}/changes/#{version_str}" + note_added = False + for line in sys.stdin: + if line.endswith("\n"): + line = line[:-1] + if line.startswith("#"): + print(line.split(" {: #")[0]) + if not note_added and version_str in line: + print(NOTE.format(docs_url=docs_url)) + note_added = True + else: + print(line) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 2141ebc55..fbb5d59d0 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -1,3 +1,12 @@ +#!/bin/env python3 +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# ] +# /// + # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use @@ -6,28 +15,40 @@ # For more info visit https://github.com/pulp/plugin_template import itertools +import json import os import re +import tomllib +import urllib.request +from pathlib import Path -import toml from git import GitCommandError, Repo -from pkg_resources import parse_version +from packaging.version import parse as parse_version + + +PYPI_PROJECT = "pulp_python" # Read Towncrier settings -tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"] +tc_settings = tomllib.loads(Path("pyproject.toml").read_text())["tool"]["towncrier"] CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( "start_string", - "\n" - if CHANGELOG_FILE.endswith(".md") - else ".. towncrier release notes start\n", + ( + "\n" + if CHANGELOG_FILE.endswith(".md") + else ".. towncrier release notes start\n" + ), ) TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})") +# Build a regex to find the header of a changelog section. +# It must have a single capture group to single out the version. +# see help(re.split) for more info. NAME_REGEX = r".*" -VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)" +VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*" +VERSION_CAPTURE_REGEX = rf"(?:YANKED )?({VERSION_REGEX})" DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" TITLE_REGEX = ( "(" @@ -35,6 +56,7 @@ TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX") ) .replace("NAME_REGEX", NAME_REGEX) + .replace("VERSION_REGEX", VERSION_CAPTURE_REGEX, 1) .replace("VERSION_REGEX", VERSION_REGEX) .replace("DATE_REGEX", DATE_REGEX) + ")" @@ -66,13 +88,27 @@ def main(): branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True) branches = [ref.name for ref in branches] + changed = False + + try: + response = urllib.request.urlopen(f"https://pypi.org/pypi/{PYPI_PROJECT}/json") + pypi_record = json.loads(response.read()) + yanked_versions = { + parse_version(version): release[0]["yanked_reason"] + for version, release in pypi_record["releases"].items() + if release[0]["yanked"] is True + } + except Exception: + # If something failed, just don't mark anything as yanked. + yanked_versions = {} + with open(CHANGELOG_FILE, "r") as f: main_changelog = f.read() preamble, main_changes = split_changelog(main_changelog) old_length = len(main_changes) for branch in branches: - print(f"Looking at branch {branch}") + print(f"Looking for './{CHANGELOG_FILE}' at branch {branch}") try: changelog = get_changelog(repo, branch) except GitCommandError: @@ -86,15 +122,25 @@ def main(): if left[0] != right[0]: main_changes.append(right) + if yanked_versions: + for change in main_changes: + if change[0] in yanked_versions and "YANKED" not in change[1].split("\n")[0]: + reason = yanked_versions[change[0]] + version = str(change[0]) + change[1] = change[1].replace(version, "YANKED " + version, count=1) + if reason: + change[1] = change[1].replace("\n", f"\n\nYank reason: {reason}\n", count=1) + changed = True + new_length = len(main_changes) - if old_length < new_length: - print(f"{new_length - old_length} new versions have been added.") + if old_length < new_length or changed: + print(f"{new_length - old_length} new versions have been added (or something has changed).") with open(CHANGELOG_FILE, "w") as fp: fp.write(preamble) for change in main_changes: fp.write(change[1]) - repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE) + repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE) if __name__ == "__main__": diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py new file mode 100755 index 000000000..0c478a212 --- /dev/null +++ b/.ci/scripts/pr_labels.py @@ -0,0 +1,60 @@ +#!/bin/env python3 + +# This script is running with elevated privileges from the main branch against pull requests. + +import re +import sys +import tomllib +from pathlib import Path + +from git import Repo + + +def main(): + assert len(sys.argv) == 3 + + with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) + BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT") + ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)") + CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$") + try: + CHANGELOG_EXTS = { + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + } + except KeyError: + CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"} + + repo = Repo(".") + + base_commit = repo.commit(sys.argv[1]) + head_commit = repo.commit(sys.argv[2]) + + pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}")) + + labels = { + "multi-commit": len(pr_commits) > 1, + "cherry-pick": False, + "no-issue": False, + "no-changelog": False, + "wip": False, + } + for commit in pr_commits: + labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None + no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None + labels["no-issue"] |= no_issue + cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None + labels["cherry-pick"] |= cherry_pick + changelog_snippets = [ + k + for k in commit.stats.files + if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS + ] + labels["no-changelog"] |= not changelog_snippets + + print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v))) + print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v))) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/skip_tests.py b/.ci/scripts/skip_tests.py new file mode 100755 index 000000000..a68d000d6 --- /dev/null +++ b/.ci/scripts/skip_tests.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +skip_tests.py - Check if only documentation files were changed in a git branch + +Usage: + ./skip_tests.py + +Arguments: + git_root: The root directory of the git project + reference_branch: The branch to compare against + +Returns: + 0: Skip + 1: NoSkip + *: Error +""" + +import sys +import os +import re +import git +import textwrap +import argparse + +DOC_PATTERNS = [ + r"^docs/", + r"\.md$", + r"LICENSE.*", + r"CHANGELOG.*", + r"CHANGES.*", + r"CONTRIBUTING.*", +] + +# Exit codes +CODE_SKIP = 0 +CODE_NO_SKIP = 1 +CODE_ERROR = 2 + + +def main() -> int: + git_root, reference_branch = get_args() + changed_files = get_changed_files(git_root, reference_branch) + if not changed_files: + return CODE_SKIP + doc_files = [f for f in changed_files if is_doc_file(f)] + not_doc_files = set(changed_files) - set(doc_files) + print_changes(doc_files, not_doc_files) + if not_doc_files: + return CODE_NO_SKIP + else: + return CODE_SKIP + + +# Utils + + +def get_changed_files(git_root: str, reference_branch: str) -> list[str]: + """Get list of files changed between current branch and reference branch.""" + repo = git.Repo(git_root) + diff_index = repo.git.diff("--name-only", reference_branch).strip() + if not diff_index: + return [] + return [f.strip() for f in diff_index.split("\n") if f.strip()] + + +def is_doc_file(file_path: str) -> bool: + """Check if a file is a documentation file.""" + for pattern in DOC_PATTERNS: + if re.search(pattern, file_path): + return True + return False + + +def print_changes(doc_files: list[str], not_doc_files: list[str]) -> None: + display_doc = " \n".join(doc_files) + print(f"doc_files({len(doc_files)})") + if doc_files: + display_doc = "\n".join(doc_files) + print(textwrap.indent(display_doc, " ")) + + print(f"non_doc_files({len(not_doc_files)})") + if not_doc_files: + display_non_doc = " \n".join(not_doc_files) + print(textwrap.indent(display_non_doc, " ")) + + +def get_args() -> tuple[str, str]: + """Parse command line arguments and validate them.""" + parser = argparse.ArgumentParser(description="Check if CI can skip tests for a git branch") + parser.add_argument("git_root", help="The root directory of the git project") + parser.add_argument("reference_branch", help="The branch to compare against") + args = parser.parse_args() + git_root = os.path.abspath(args.git_root) + ref_branch = args.reference_branch + + if not os.path.exists(git_root): + raise ValueError(f"Git root directory does not exist: {git_root}") + if not os.path.isdir(git_root): + raise ValueError(f"Git root is not a directory: {git_root}") + try: + git.Repo(git_root) + except git.InvalidGitRepositoryError: + raise ValueError(f"Directory is not a git repository: {git_root}") + return git_root, ref_branch + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + print(e) + sys.exit(CODE_ERROR) diff --git a/.ci/scripts/tweet.py b/.ci/scripts/tweet.py deleted file mode 100755 index 5033cf449..000000000 --- a/.ci/scripts/tweet.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import sys -from tweepy import Client - -release_version = sys.argv[1] -if release_version.endswith(".0"): - client = Client( - consumer_key=os.getenv("TWITTER_API_KEY"), - consumer_secret=os.getenv("TWITTER_API_KEY_SECRET"), - access_token=os.getenv("TWITTER_ACCESS_TOKEN"), - access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"), - ) - link = "https://docs.pulpproject.org/pulp_python/changes.html" - msg = f"pulp_python-{release_version} - Check out for more details: {link}" - release_msg = f"Hey! We've just released {msg}" - client.create_tweet(text=release_msg) diff --git a/.ci/scripts/update_ci_branches.py b/.ci/scripts/update_ci_branches.py deleted file mode 100755 index c82e2a072..000000000 --- a/.ci/scripts/update_ci_branches.py +++ /dev/null @@ -1,25 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import os -import sys -import requests - -branches = sys.argv[1:] - -headers = { - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github.v3+json", -} - -github_api = "https://api.github.com" -workflow_path = "/actions/workflows/update_ci.yml/dispatches" -url = f"{github_api}/repos/pulp/pulp_python{workflow_path}" - -for branch in branches: - print(f"Updating {branch}") - requests.post(url, headers=headers, json={"ref": branch}) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py old mode 100755 new mode 100644 index d6f22c71a..a4dc9004a --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -1,46 +1,31 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template +# This file is managed by the plugin template. +# Do not edit. +import os import re +import subprocess import sys +import tomllib +import yaml from pathlib import Path -import subprocess - -import os -import warnings from github import Github -NO_ISSUE = "[noissue]" -CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] -sha = sys.argv[1] -message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - - -KEYWORDS = ["fixes", "closes"] - -g = Github(os.environ.get("GITHUB_TOKEN")) -repo = g.get_repo("pulp/pulp_python") - - -def __check_status(issue): +def check_status(issue, repo, cherry_pick): gi = repo.get_issue(int(issue)) if gi.pull_request: sys.exit(f"Error: issue #{issue} is a pull request.") - if gi.closed_at and "cherry picked from commit" not in message: - warnings.warn( - "When backporting, use the -x flag to append a line that says " - "'(cherry picked from commit ...)' to the original commit message." + if gi.closed_at and not cherry_pick: + print("Make sure to use 'git cherry-pick -x' when backporting a change.") + print( + "If a backport of a change requires a significant amount of rewriting, " + "consider creating a new issue." ) sys.exit(f"Error: issue #{issue} is closed.") -def __check_changelog(issue): +def check_changelog(issue, CHANGELOG_EXTS): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -48,31 +33,65 @@ def __check_changelog(issue): for match in matches: if match.suffix not in CHANGELOG_EXTS: sys.exit(f"Invalid extension for changelog entry '{match}'.") - if match.suffix == ".feature" and "cherry picked from commit" in message: - sys.exit(f"Can not backport '{match}' as it is a feature.") - - -print("Checking commit message for {sha}.".format(sha=sha[0:7])) - -# validate the issue attached to the commit -regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -pattern = re.compile(regex, re.IGNORECASE) - -issues = pattern.findall(message) - -if issues: - for issue in pattern.findall(message): - __check_status(issue) - __check_changelog(issue) -else: - if NO_ISSUE in message: - print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) - elif "Merge" in message and "cherry picked from commit" in message: - pass - else: - sys.exit( - "Error: no attached issues found for {sha}. If this was intentional, add " - " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) - ) -print("Commit message for {sha} passed.".format(sha=sha[0:7])) + +def main() -> None: + TEMPLATE_CONFIG = yaml.safe_load(Path("template_config.yml").read_text()) + GITHUB_ORG = TEMPLATE_CONFIG["github_org"] + PLUGIN_NAME = TEMPLATE_CONFIG["plugin_name"] + + with Path("pyproject.toml").open("rb") as _fp: + PYPROJECT_TOML = tomllib.load(_fp) + KEYWORDS = ["fixes", "closes"] + BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"^fixup!", # This is created by 'git commit --fixup' + r"Apply suggestions from code review", # This usually comes from GitHub + ] + try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] + except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] + NOISSUE_MARKER = "[noissue]" + + sha = sys.argv[1] + message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") + + if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + + blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] + if blocking_matches: + print("Found these phrases in the commit message:") + for m in blocking_matches: + print(" - " + m.group(0)) + sys.exit("This PR is not ready for consumption.") + + g = Github(os.environ.get("GITHUB_TOKEN")) + repo = g.get_repo(f"{GITHUB_ORG}/{PLUGIN_NAME}") + + print("Checking commit message for {sha}.".format(sha=sha[0:7])) + + # validate the issue attached to the commit + issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords="|".join(KEYWORDS)) + issues = re.findall(issue_regex, message, re.IGNORECASE) + cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" + cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) + + if issues: + for issue in issues: + check_status(issue, repo, cherry_pick) + check_changelog(issue, CHANGELOG_EXTS) + + print("Commit message for {sha} passed.".format(sha=sha[0:7])) + + +if __name__ == "__main__": + main() diff --git a/.flake8 b/.flake8 index 64403dceb..9e00b3faa 100644 --- a/.flake8 +++ b/.flake8 @@ -8,13 +8,14 @@ exclude = ./docs/*,*/migrations/* per-file-ignores = */__init__.py: F401 -ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402 +ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 max-line-length = 100 # Flake8 builtin codes # -------------------- # E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed # W503: This enforces operators before line breaks which is not pep8 or black compatible. +# F824: 'nonlocal' is unused: name is never assigned in scope # Flake8-quotes extension codes # ----------------------------- diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..4bee60faf --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,15 @@ + + +### 📜 Checklist + +- [ ] Commits are cleanly separated with meaningful messages (simple features and bug fixes should be [squashed](https://pulpproject.org/pulpcore/docs/dev/guides/git/#rebasing-and-squashing) to one commit) +- [ ] A [changelog entry](https://pulpproject.org/pulpcore/docs/dev/guides/git/#changelog-update) or entries has been added for any significant changes +- [ ] Follows the [Pulp policy on AI Usage](https://pulpproject.org/help/more/governance/ai_policy/) +- [ ] (For new features) - User documentation and test coverage has been added + +See: [Pull Request Walkthrough](https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/) diff --git a/.github/template_gitref b/.github/template_gitref deleted file mode 100644 index cd7b27530..000000000 --- a/.github/template_gitref +++ /dev/null @@ -1 +0,0 @@ -2021.08.26-250-gf4315cf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..988902a0f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,111 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Build" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + build: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + - name: "Install python dependencies" + run: | + pip install build packaging twine wheel mkdocs jq + - name: "Build package" + run: | + python3 -m build + twine check dist/* + - name: "Install built packages" + run: | + pip install dist/pulp_python-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt + - name: "Generate api specs" + run: | + pulpcore-manager openapi --file "api.json" + pulpcore-manager openapi --bindings --component "python" --file "python-api.json" + - name: "Upload Package whl" + uses: "actions/upload-artifact@v5" + with: + name: "plugin_package" + path: "pulp_python/dist/" + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload API specs" + uses: "actions/upload-artifact@v5" + with: + name: "api_spec" + path: | + pulp_python/api.json + pulp_python/python-api.json + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Python bindings packages" + run: | + .github/workflows/scripts/build_python_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v5" + with: + name: "python-client.tar" + path: | + pulp_python/python-python-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload python client docs" + uses: "actions/upload-artifact@v5" + with: + name: "python-client-docs.tar" + path: | + pulp_python/python-python-client-docs.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Ruby bindings packages" + run: | + .github/workflows/scripts/build_ruby_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload Ruby client" + uses: "actions/upload-artifact@v5" + with: + name: "ruby-client.tar" + path: | + pulp_python/python-ruby-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true +... diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml deleted file mode 100644 index 621ff7324..000000000 --- a/.github/workflows/changelog.yml +++ /dev/null @@ -1,58 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - ---- -name: Python changelog update -on: - push: - branches: - - main - paths: - - CHANGES.rst - - CHANGES.md - workflow_dispatch: - -jobs: - - update-changelog: - runs-on: ubuntu-latest - strategy: - fail-fast: false - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install python dependencies - run: | - echo ::group::PYDEPS - pip install -r doc_requirements.txt - echo ::endgroup:: - - - name: Fake api schema - run: | - mkdir -p docs/_build/html - echo "{}" > docs/_build/html/api.json - mkdir -p docs/_static - echo "{}" > docs/_static/api.json - - name: - run: | - pip install "Jinja2<3.1" - make diagrams html - working-directory: ./docs - env: - PULP_CONTENT_ORIGIN: "http://localhost/" - - - name: Publish changlog to pulpproject.org - run: .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} - env: - PULP_DOCS_KEY: ${{ secrets.PULP_DOCS_KEY }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12b8db8b6..2965afca1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,218 +6,169 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Python CI +name: "Python CI" on: {pull_request: {branches: ['*']}} concurrency: group: ${{ github.ref_name }}-${{ github.workflow }} cancel-in-progress: true -jobs: +defaults: + run: + working-directory: "pulp_python" - ready-to-ship: - runs-on: ubuntu-latest +jobs: + check-commits: + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v6" with: fetch-depth: 0 - - uses: actions/setup-python@v4 + path: "pulp_python" + - uses: "actions/setup-python@v6" with: - python-version: "3.8" - - name: Install requirements - run: pip3 install github - - name: Check commit message + python-version: "3.11" + - name: "Install python dependencies" + run: | + pip install requests pygithub pyyaml + - name: "Check commit message" if: github.event_name == 'pull_request' env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - run: sh .github/workflows/scripts/check_commit.sh - - name: Verify requirements files - run: python .ci/scripts/check_requirements.py - single_commit: - runs-on: ubuntu-latest - name: Assert single commit - if: github.base_ref == 'main' + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + run: | + .github/workflows/scripts/check_commit.sh + + check-changes: + runs-on: "ubuntu-latest" + outputs: + run_tests: "${{ steps.check.outputs.run_tests }}" + run_docs: "${{ steps.check.outputs.run_docs }}" steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v6" with: fetch-depth: 0 - - name: Checkout main - run: git fetch origin main - - name: create local main branch - run: git branch main origin/main - - name: Commit Count Check - run: test `git log --oneline --no-merges HEAD ^main | wc -l ` = 1 - - lint: - runs-on: ubuntu-latest + path: "pulp_python" - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v6" with: - python-version: "3.8" - # lint_requirements contains tools needed for flake8, etc. - - name: Install requirements - run: pip3 install -r lint_requirements.txt - - + python-version: "3.12" - # Lint code. - - name: Run flake8 - run: flake8 + - name: "Install python dependencies" + run: | + pip install gitpython - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + - name: "Analyze changed files" + id: "check" + shell: "bash" + run: | + # We only test docs on the default branch (usually main) + if [[ "${{ github.base_ref }}" == *"main" ]]; then + echo "run_docs=1" >> $GITHUB_OUTPUT + else + echo "run_docs=0" >> $GITHUB_OUTPUT + fi + + set +e + BASE_REF=${{ github.event.pull_request.base.sha }} + echo "Checking against:" + git name-rev $BASE_REF + python3 .ci/scripts/skip_tests.py . $BASE_REF + exit_code=$? + if [ $exit_code -ne 0 ] && [ $exit_code -ne 1 ]; then + echo "Error: skip_tests.py returned unexpected exit code $exit_code" + exit $exit_code + fi + echo "run_tests=$exit_code" >> $GITHUB_OUTPUT + + docs: + needs: "check-changes" + uses: "./.github/workflows/docs.yml" + with: + run_docs: "${{ needs.check-changes.outputs.run_docs }}" - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + lint: + uses: "./.github/workflows/lint.yml" - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + sanity: + uses: "./.github/workflows/sanity.yml" - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + build: + needs: + - "check-changes" + - "lint" + if: "needs.check-changes.outputs.run_tests == '1'" + uses: "./.github/workflows/build.yml" test: - runs-on: ubuntu-latest - # run only after lint finishes - needs: lint - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - TEST: lowerbounds - outputs: - deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} - deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} - deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} - deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }} + needs: "build" + uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] + deprecations: + runs-on: "ubuntu-latest" + if: "github.base_ref == 'main'" + needs: "test" steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install httpie + - name: "Create working directory" run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables + mkdir -p "pulp_python" + working-directory: "." + - name: "Download Deprecations" + uses: "actions/download-artifact@v8" + with: + pattern: "deprecations-*" + path: "pulp_python" + merge-multiple: true + - name: "Print deprecations" run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - if: github.event_name != 'pull_request' - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} + cat deprecations-*.txt | sort -u + ! cat deprecations-*.txt | grep '[^[:space:]]' - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Extract Deprecations from Logs - id: deprecations - run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - deprecations: - runs-on: ubuntu-latest - if: github.base_ref == 'main' - needs: test + ready-to-ship: + # This is a dummy dependent task to have a single entry for the branch protection rules. + runs-on: "ubuntu-latest" + needs: + - "check-changes" + - "check-commits" + - "lint" + - "test" + - "docs" + - "sanity" + if: "always()" steps: - - name: Fail on deprecations - run: | - test -z "${{ needs.test.outputs.deprecations-pulp }}" - test -z "${{ needs.test.outputs.deprecations-azure }}" - test -z "${{ needs.test.outputs.deprecations-s3 }}" - test -z "${{ needs.test.outputs.deprecations-lowerbounds }}" - - name: Print deprecations - if: failure() + - name: "Collect needed jobs results" + working-directory: "." run: | - echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-lowerbounds }}" | base64 -d - - + RUN_TESTS=${{ needs.check-changes.outputs.run_tests }} + RUN_DOCS=${{ needs.check-changes.outputs.run_docs }} + + check_jobs() { + local filter="$1" + local needs_json='${{toJson(needs)}}' + # output failed jobs after filter + echo "$needs_json" | jq -r "to_entries[]|select($filter)|select(.value.result!=\"success\")|.key + \": \" + .value.result" + # fails if not all selected jobs passed + echo "$needs_json" | jq -e "to_entries|map(select($filter))|map(select(.value.result!=\"success\"))|length == 0" + } + + if [ "$RUN_TESTS" == "1" ] && [ "$RUN_DOCS" == "1" ]; then + FILTERS="true" # check all jobs + elif [ "$RUN_TESTS" == "1" ] && [ "$RUN_DOCS" == "0" ]; then + echo "Skipping docs: running on non-default branch" + FILTERS='.key != "docs"' + elif [ "$RUN_TESTS" == "0" ] && [ "$RUN_DOCS" == "1" ]; then + echo "Skipping tests: only doc changes" + FILTERS='.key != "lint" and .key != "test"' + else # RUN_TESTS=0, RUN_DOCS=0 + echo "What is this PR doing??" + FILTERS='.key != "lint" and .key != "test" and .key != "docs"' + fi + + check_jobs "$FILTERS" + echo "CI says: Looks good!" +... diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1dff2755d..22e5d99b8 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,39 +4,42 @@ # './plugin-template --github pulp_python' to update this file. # # For more info visit https://github.com/pulp/plugin_template +--- name: "Python CodeQL" on: workflow_dispatch: schedule: - - cron: '37 1 * * 6' + - cron: "37 1 * * 6" concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: analyze: - name: Analyze - runs-on: ubuntu-latest + name: "Analyze" + runs-on: "ubuntu-latest" permissions: - actions: read - contents: read - security-events: write + actions: "read" + contents: "read" + security-events: "write" strategy: fail-fast: false matrix: - language: [ 'python' ] + language: + - "python" steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: "Checkout repository" + uses: "actions/checkout@v6" - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} + - name: "Initialize CodeQL" + uses: "github/codeql-action/init@v4" + with: + languages: "${{ matrix.language }}" - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: "Perform CodeQL Analysis" + uses: "github/codeql-action/analyze@v4" +... diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 19a5faad6..6b57c7854 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Create New Release Branch +name: "Create New Release Branch" on: workflow_dispatch: @@ -15,39 +15,47 @@ env: jobs: create-branch: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false + permissions: + contents: "write" + steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v6" with: fetch-depth: 0 - path: pulp_python + path: "pulp_python" + + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + repository: "pulp/plugin_template" + path: "plugin_template" - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v6" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install bump2version jinja2 pyyaml - echo ::endgroup:: + pip install bump-my-version packaging -r plugin_template/requirements.txt - - name: Setting secrets - working-directory: pulp_python - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + - name: "Setting secrets" + working-directory: "pulp_python" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} + SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: Determine new branch name - working-directory: pulp_python + - name: "Determine new branch name" + working-directory: "pulp_python" run: | # Just to be sure... git checkout main - NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')" + NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')" if [ -z "$NEW_BRANCH" ] then echo Could not determine the new branch name. @@ -55,38 +63,43 @@ jobs: fi echo "NEW_BRANCH=${NEW_BRANCH}" >> "$GITHUB_ENV" - - name: Create release branch - working-directory: pulp_python + - name: "Create release branch" + working-directory: "pulp_python" run: | git branch "${NEW_BRANCH}" - - name: Bump version on main branch - working-directory: pulp_python + - name: "Bump version on main branch" + working-directory: "pulp_python" run: | - bump2version --no-commit minor + bump-my-version bump --no-commit minor - - name: Remove entries from CHANGES directory - working-directory: pulp_python + - name: "Remove entries from CHANGES directory" + working-directory: "pulp_python" run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + - - name: Make a PR with version bump and without CHANGES/* - uses: peter-evans/create-pull-request@v4 + - name: "Update CI branches in template_config" + working-directory: "plugin_template" + run: | + python3 ./plugin-template pulp_python --github --latest-release-branch "${NEW_BRANCH}" + git add -A + + - name: "Make a PR with version bump and without CHANGES/*" + uses: "peter-evans/create-pull-request@v8" with: - path: pulp_python - token: ${{ secrets.RELEASE_TOKEN }} - committer: pulpbot - author: pulpbot - branch: minor-version-bump - base: main - title: Bump minor version - body: '[noissue]' + path: "pulp_python" + token: "${{ secrets.RELEASE_TOKEN }}" + committer: "pulpbot " + author: "pulpbot " + branch: "minor-version-bump" + base: "main" + title: "Bump minor version" commit-message: | Bump minor version - [noissue] delete-branch: true - - name: Push release branch - working-directory: pulp_python + - name: "Push release branch" + working-directory: "pulp_python" run: | git push origin "${NEW_BRANCH}" +... diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..c897fcac2 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,43 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Docs CI" +on: + workflow_call: + inputs: + run_docs: + description: "Whether to run docs jobs" + required: true + type: "string" + +jobs: + changelog: + runs-on: "ubuntu-latest" + defaults: + run: + working-directory: "pulp_python" + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + - uses: "actions/setup-python@v6" + with: + python-version: "3.12" + - name: "Install python dependencies" + run: | + pip install towncrier + - name: "Build changelog" + run: | + towncrier build --yes --version 4.0.0.ci + docs: + if: "${{ inputs.run_docs == '1' }}" + uses: "pulp/pulp-docs/.github/workflows/docs-ci.yml@main" + with: + pulpdocs_ref: "main" +... diff --git a/.github/workflows/kanban.yml b/.github/workflows/kanban.yml deleted file mode 100644 index 97e750080..000000000 --- a/.github/workflows/kanban.yml +++ /dev/null @@ -1,103 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -# Manage issues in a project board using https://github.com/leonsteinhaeuser/project-beta-automations - ---- -name: Kanban -on: - pull_request_target: - issues: - types: - - labeled - - reopened - - assigned - - closed - -env: - free_to_take: Free to take - in_progress: In Progress - needs_review: Needs review - done: Done - -jobs: - # only prio-list labeled items should be added to the board - add-to-project-board: - if: github.event_name == 'issues' && contains(github.event.issue.labels.*.name, 'prio-list') && contains(fromJson('["labeled", "reopened"]'), github.event.action) - runs-on: ubuntu-latest - steps: - - name: Add issue to Free-to-take list - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.free_to_take }} # Target status - - move-to-inprogress: - if: github.event_name == 'issues' && github.event.action == 'assigned' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the In Progress column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.in_progress }} # Target status - - find-linked-issues: - if: github.event_name == 'pull_request_target' - runs-on: ubuntu-latest - name: Find issues linked to a PR - outputs: - linked-issues: ${{ steps.linked-issues.outputs.issues }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Get Linked Issues Action - uses: kin/gh-action-get-linked-issues@v1.0 - id: linked-issues - with: - access-token: ${{ secrets.RELEASE_TOKEN }} - - move-to-needs-review: - if: github.event_name == 'pull_request_target' && contains(fromJson(needs.find-linked-issues.outputs.linked-issues).*.issue.state, 'open') - runs-on: ubuntu-latest - name: Move linked issues to Needs Review - needs: find-linked-issues - strategy: - max-parallel: 3 - matrix: - issues: ${{ fromJSON(needs.find-linked-issues.outputs.linked-issues) }} - steps: - - name: Move to Needs Review - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ matrix.issues.issue.node_id }} - operation_mode: status - status_value: ${{ env.needs_review }} # Target status - - move-to-done: - if: github.event_name == 'issues' && github.event.action == 'closed' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the Done column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.done }} # Target status diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..d07a39a08 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,51 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Lint" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + lint: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + pip install -r lint_requirements.txt + + - name: "Lint workflow files" + run: | + yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + + # Lint code. + - name: "Run flake8" + run: | + flake8 + + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh + + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh +... diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 66862def0..ed8149ce9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,314 +6,74 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Python Nightly CI +name: "Python Nightly CI" on: schedule: # * is a special character in YAML so you have to quote this string # runs at 3:00 UTC daily - - cron: '00 3 * * *' + - cron: "00 3 * * *" workflow_dispatch: +defaults: + run: + working-directory: "pulp_python" + concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: - test: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - - TEST: generate-bindings - - TEST: lowerbounds - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - uses: ruby/setup-ruby@v1 - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - with: - ruby-version: "2.6" - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Upload python client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client.tar - path: python-client.tar - - - name: Upload python client docs - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client-docs.tar - path: python-client-docs.tar - - - name: Upload ruby client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: ruby-client.tar - path: ruby-client.tar - - name: Upload built docs - if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v3 - with: - name: docs.tar - path: docs/docs.tar + build: + uses: "./.github/workflows/build.yml" - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + test: + needs: "build" + uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] changelog: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v6" with: fetch-depth: 0 + path: "pulp_python" - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v6" with: - python-version: "3.11" + python-version: "3.13" - - name: Install python dependencies + - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install gitpython toml - echo ::endgroup:: + pip install gitpython packaging toml - - name: Configure Git with pulpbot name and email + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Collect changes from all branches - run: python .ci/scripts/collect_changes.py + - name: "Collect changes from all branches" + run: | + python .ci/scripts/collect_changes.py - - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 + - name: "Create Pull Request" + uses: "peter-evans/create-pull-request@v8" + id: "create_pr_changelog" with: - token: ${{ secrets.RELEASE_TOKEN }} - title: 'Update Changelog' - body: '' - branch: 'changelog/update' + token: "${{ secrets.RELEASE_TOKEN }}" + title: "Update Changelog" + body: "" + branch: "changelog/update" delete-branch: true - - publish: - runs-on: ubuntu-latest - needs: test - - env: - TEST: publish - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Install python dependencies + path: "pulp_python" + - name: "Mark PR automerge" + working-directory: "pulp_python" run: | - echo ::group::PYDEPS - pip install wheel - echo ::endgroup:: - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + gh pr merge --rebase --auto "${{ steps.create_pr_changelog.outputs.pull-request-number }}" + if: "steps.create_pr_changelog.outputs.pull-request-number" env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - - name: Download built docs - uses: actions/download-artifact@v3 - with: - name: docs.tar - - - name: Download Python client docs - uses: actions/download-artifact@v3 - with: - name: python-client-docs.tar - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar -C ./docs - .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} - - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true +... diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml new file mode 100644 index 000000000..d80548961 --- /dev/null +++ b/.github/workflows/pr_checks.yml @@ -0,0 +1,73 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Python PR static checks" +on: + pull_request_target: + types: + - "opened" + - "synchronize" + - "reopened" + branches: + - "main" + - "[0-9]+.[0-9]+" + +# This workflow runs with elevated permissions. +# Do not even think about running a single bit of code from the PR. +# Static analysis should be fine however. + +concurrency: + group: "${{ github.event.pull_request.number }}-${{ github.workflow }}" + cancel-in-progress: true + +jobs: + apply_labels: + runs-on: "ubuntu-latest" + name: "Label PR" + permissions: + pull-requests: "write" + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 0 + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + - name: "Determine PR labels" + run: | + pip install GitPython==3.1.42 + git fetch origin ${{ github.event.pull_request.head.sha }} + python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" + - uses: "actions/github-script@v8" + name: "Apply PR Labels" + with: + script: | + const { ADD_LABELS, REMOVE_LABELS } = process.env; + + if (REMOVE_LABELS.length) { + for await (const labelName of REMOVE_LABELS.split(",")) { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } + } + } + if (ADD_LABELS.length) { + await github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ADD_LABELS.split(","), + }); + } +... diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..6d0d3aa8c --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,159 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Python Publish Release" +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + +defaults: + run: + working-directory: "pulp_python" + +jobs: + build: + uses: "./.github/workflows/build.yml" + publish-package: + runs-on: "ubuntu-latest" + needs: + - "build" + environment: + name: "pypi" + url: "https://pypi.org/p/pulp-python" + permissions: + id-token: "write" + + steps: + - uses: "actions/download-artifact@v4" + with: + name: "plugin_package" + path: "dist/" + + - name: "Publish package to PyPI" + uses: pypa/gh-action-pypi-publish@release/v1 + publish-python-bindings: + runs-on: "ubuntu-latest" + needs: + - "build" + environment: + name: "pypi" + permissions: + id-token: "write" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + + - name: "Download Python client" + uses: "actions/download-artifact@v4" + with: + name: "python-client.tar" + path: "pulp_python/" + + - name: "Untar python client packages" + run: | + tar -xvf python-python-client.tar + + - name: "Publish client to pypi" + uses: "pypa/gh-action-pypi-publish@release/v1" + with: + packages-dir: "pulp_python/dist/" + publish-ruby-bindings: + runs-on: "ubuntu-latest" + needs: + - "build" + environment: + name: "rubygems" + permissions: + id-token: "write" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + + - name: "Download Ruby client" + uses: "actions/download-artifact@v4" + with: + name: "ruby-client.tar" + path: "pulp_python/" + + - name: "Untar Ruby client packages" + run: | + tar -xvf python-ruby-client.tar + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: "Set RubyGems Credentials" + uses: "rubygems/configure-rubygems-credentials@v1.0.0" + + - name: "Publish client to RubyGems" + run: | + gem push "pulp_python_client-${{ github.ref_name }}.gem" + + create-gh-release: + runs-on: "ubuntu-latest" + needs: + - "build" + - "publish-package" + - "publish-python-bindings" + - "publish-ruby-bindings" + + permissions: + contents: "write" + + env: + TAG_NAME: "${{ github.ref_name }}" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 0 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Install towncrier" + run: | + pip install towncrier + + - name: "Get release notes" + id: "get_release_notes" + shell: "bash" + run: | + # The last commit before the release commit contains the release CHANGES fragments + git checkout "${TAG_NAME}~" + NOTES=$(towncrier build --draft --version $TAG_NAME | .ci/scripts/clean_gh_release_notes.py pulp_python $TAG_NAME) + echo "body<> $GITHUB_OUTPUT + echo "$NOTES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: "Create release on GitHub" + uses: "actions/github-script@v8" + env: + RELEASE_BODY: "${{ steps.get_release_notes.outputs.body }}" + with: + script: | + const { TAG_NAME, RELEASE_BODY } = process.env; + + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: TAG_NAME, + body: RELEASE_BODY, + make_latest: "legacy", + }); +... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 85224a883..e056bcfaa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,282 +9,51 @@ name: Python Release Pipeline on: workflow_dispatch: - inputs: - release: - description: "Release tag (e.g. 3.2.1)" - before_script: - description: | - Bash code to run before bindings and docs are built. This should only be used when re-running - a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst - required: false -env: - RELEASE_WORKFLOW: true +defaults: + run: + working-directory: "pulp_python" jobs: build-artifacts: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v6" with: fetch-depth: 0 + path: "pulp_python" + token: ${{ secrets.RELEASE_TOKEN }} - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v6" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install packaging~=21.3 bump2version gitpython towncrier==19.9.0 wheel requests - echo ::endgroup:: + pip install bump-my-version towncrier - - name: Configure Git with pulpbot name and email + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Create the release commit, tag it, create a post-release commit, and build plugin package - run: python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} - - - name: 'Tar files' - run: tar -cvf pulp_python.tar . - - - name: 'Upload Artifact' - uses: actions/upload-artifact@v3 - with: - name: pulp_python.tar - path: pulp_python.tar - - - build-bindings-docs: - needs: build-artifacts - runs-on: ubuntu-latest - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: docs - - steps: - - uses: actions/download-artifact@v3 - with: - name: pulp_python.tar - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: Untar repository - run: | - tar -xf pulp_python.tar - - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - - name: Install httpie + - name: "Setting secrets" run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: Install + - name: "Tag the release" run: | - export PLUGIN_VERSION=${{ github.event.inputs.release }} - .github/workflows/scripts/install.sh + .github/workflows/scripts/release.sh + shell: "bash" env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - shell: bash - - - name: Additional before_script - run: ${{ github.event.inputs.before_script }} - shell: bash - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - name: Install Ruby client - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Upload python client packages - uses: actions/upload-artifact@v3 - with: - name: python-client.tar - path: python-client.tar - - - name: Upload python client docs - uses: actions/upload-artifact@v3 - with: - name: python-client-docs.tar - path: python-client-docs.tar - - name: Upload ruby client packages - uses: actions/upload-artifact@v3 - with: - name: ruby-client.tar - path: ruby-client.tar - - name: Build docs - run: | - export DJANGO_SETTINGS_MODULE=pulpcore.app.settings - export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py - make -C docs/ PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" diagrams html - tar -cvf docs/docs.tar docs/_build - - - name: Upload built docs - uses: actions/upload-artifact@v3 - with: - name: docs.tar - path: docs/docs.tar - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - - - publish: - runs-on: ubuntu-latest - needs: build-bindings-docs - - env: - TEST: publish - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - steps: - - uses: actions/download-artifact@v3 - with: - name: pulp_python.tar - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: Configure Git with pulpbot name and email - run: | - git config --global user.name 'pulpbot' - git config --global user.email 'pulp-infra@redhat.com' - - - name: Untar repository - run: | - tar -xf pulp_python.tar - - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install python dependencies - run: | - echo ::group::PYDEPS - pip install gitpython requests packaging~=21.3 tweepy - echo ::endgroup:: - - - name: Push branch and tag to GitHub - run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - - name: Download built docs - uses: actions/download-artifact@v3 - with: - name: docs.tar - - - name: Download Python client docs - uses: actions/download-artifact@v3 - with: - name: python-client-docs.tar - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }} - - name: Deploy plugin to pypi - run: bash .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} - - name: Download Python client - uses: actions/download-artifact@v3 - with: - name: python-client.tar - - - name: Untar python client packages - run: tar -xvf python-client.tar - - - name: Publish client to pypi - run: bash .github/workflows/scripts/publish_client_pypi.sh - - name: Download Ruby client - uses: actions/download-artifact@v3 - with: - name: ruby-client.tar - - - name: Untar Ruby client packages - run: tar -xvf ruby-client.tar - - - name: Publish client to rubygems - run: bash .github/workflows/scripts/publish_client_gem.sh - - - name: Update GitHub - continue-on-error: true - run: | - set -euv - export COMMIT_MSG=$(git log --format=%B --no-merges -1) - export GH_ISSUES=$(echo $COMMIT_MSG | grep -o "GH Issues: .*" | awk '{print $3}') - pip install pygithub - - echo "GH Issues $GH_ISSUES" - python .ci/scripts/update_github.py - - - name: Create release on GitHub - run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" +... diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml new file mode 100644 index 000000000..223f07cbd --- /dev/null +++ b/.github/workflows/sanity.yml @@ -0,0 +1,54 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + + +# This file describes checks that should prevent a premature merge, +# but still let the tests run for demonstrations or experiments. +--- +name: "Sanity" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + sanity: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + pip install -r lint_requirements.txt + + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest + + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py + + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh +... diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 474720c19..782c6ae8d 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -7,79 +7,126 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script prepares the scenario definition in the .ci/ansible/vars/main.yaml file. +# +# It requires the following environment: +# TEST - The name of the scenario to prepare. +# +# It may also dump the {lower,upper}bounds_constraints.txt for the specific scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -mveuo pipefail +if [ -f .github/workflows/scripts/pre_before_install.sh ]; then + source .github/workflows/scripts/pre_before_install.sh +fi -if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ] -then - BRANCH_BUILD=0 -else - BRANCH_BUILD=1 - BRANCH="${GITHUB_REF##refs/heads/}" +COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" +COMPONENT_SOURCE="./pulp_python/dist/pulp_python-${COMPONENT_VERSION}-py3-none-any.whl" +if [ "$TEST" = "s3" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" fi -if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ] -then - TAG_BUILD=0 -else - TAG_BUILD=1 - BRANCH="${GITHUB_REF##refs/tags/}" +if [ "$TEST" = "azure" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]" fi -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export COMMIT_MSG - -COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) - -mkdir .ci/ansible/vars || true -echo "---" > .ci/ansible/vars/main.yaml -echo "legacy_component_name: pulp_python" >> .ci/ansible/vars/main.yaml -echo "component_name: python" >> .ci/ansible/vars/main.yaml -echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml - -export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh -export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh - -if [ -f $PRE_BEFORE_INSTALL ]; then - source $PRE_BEFORE_INSTALL +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt fi - -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 +if [[ "$TEST" = "lowerbounds" ]]; then + python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi -if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] -then - echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml +export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") + +echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" + +# Compose the scenario definition. +mkdir -p .ci/ansible/vars + +cat > .ci/ansible/vars/main.yaml << VARSYAML +--- +scenario: "${TEST}" +plugin_name: "pulp_python" +legacy_component_name: "pulp_python" +component_name: "python" +component_version: "${COMPONENT_VERSION}" +pulp_env: {} +pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} +pulp_scheme: "https" +api_root: "${PULP_API_ROOT}" +image: + name: "pulp" + tag: "ci_build" + ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest" + source: "${COMPONENT_SOURCE}" + ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) + upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) + lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) + webserver_snippet: $(test -f pulp_python/app/webserver_snippets/nginx.conf && echo -n true || echo -n false ) +extra_files: + - origin: "pulp_python" + destination: "pulp_python" +services: + - name: "pulp" + image: "pulp:ci_build" + volumes: + - "./settings:/etc/pulp" + - "../../../pulp-openapi-generator:/root/pulp-openapi-generator" + env: + PULP_WORKERS: "4" + PULP_HTTPS: "true" +VARSYAML + +if [ "$TEST" = "s3" ]; then + MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA + MINIO_SECRET_KEY=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 + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "minio" + image: "minio/minio" + env: + MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}" + MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}" + command: "server /data" +s3_test: true +minio_access_key: "${MINIO_ACCESS_KEY}" +minio_secret_key: "${MINIO_SECRET_KEY}" +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}} +pulp_scenario_env: {} +VARSYAML fi - -cd .. - -git clone --depth=1 https://github.com/pulp/pulp-openapi-generator.git - -# Intall requirements for ansible playbooks -pip install docker netaddr boto3 ansible - -for i in {1..3} -do - ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3 -done -if [[ $s -gt 0 ]] -then - echo "Failed to install amazon.aws" - exit $s +if [ "$TEST" = "azure" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-azurite" + image: "mcr.microsoft.com/azure-storage/azurite" + command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0" +azure_test: true +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}} +pulp_scenario_env: {} +VARSYAML fi -cd pulp_python - -if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_requirements.txt - mv lowerbounds_requirements.txt requirements.txt +if [ "$TEST" = "gcp" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-gcp" + image: "fsouza/fake-gcs-server" + volumes: + - "storage_data:/etc/pulp" + command: " -scheme http" +gcp_test: true +pulp_scenario_settings: null +pulp_scenario_env: {} +VARSYAML fi -if [ -f $POST_BEFORE_INSTALL ]; then - source $POST_BEFORE_INSTALL +cat >> .ci/ansible/vars/main.yaml << VARSYAML +... +VARSYAML +cat .ci/ansible/vars/main.yaml + +if [ -f .github/workflows/scripts/post_before_install.sh ]; then + source .github/workflows/scripts/post_before_install.sh fi diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 175db1fe9..13809a87a 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -7,50 +7,46 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script dumps some files to help understand the setup of the test scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -euv - source .github/workflows/scripts/utils.sh -export PRE_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/pre_before_script.sh -export POST_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/post_before_script.sh - -if [[ -f $PRE_BEFORE_SCRIPT ]]; then - source $PRE_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/pre_before_script.sh ]]; then + source .github/workflows/scripts/pre_before_script.sh fi -# Developers should be able to reproduce the containers with this config -echo "CI vars:" -tail -v -n +1 .ci/ansible/vars/main.yaml - # Developers often want to know the final pulp config -echo "PULP CONFIG:" -tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json +echo +echo "# Pulp config:" +tail -v -n +1 .ci/ansible/settings/settings.* -echo "Containerfile:" +echo +echo "# Containerfile:" tail -v -n +1 .ci/ansible/Containerfile -# Needed for some functional tests -cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" -cmd_prefix bash -c "usermod -a -G wheel pulp" +echo +echo "# Constraints Files:" +# They need not even exist. +tail -v -n +1 ../*/*constraints.txt || true -SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds") -if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then - # Many functional tests require these - cmd_prefix dnf install -yq lsof which -fi +echo +echo "# pip list outside the container" +pip list -if [[ "${REDIS_DISABLED:-false}" == true ]]; then - cmd_prefix bash -c "s6-rc -d change redis" - echo "The Redis service was disabled for $TEST" -fi +echo +echo "# pip list inside the container" +cmd_prefix bash -c "pip3 list" + +echo +echo "# State of the containers" +docker ps -a -if [[ -f $POST_BEFORE_SCRIPT ]]; then - source $POST_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/post_before_script.sh ]]; then + source .github/workflows/scripts/post_before_script.sh fi -# Lots of plugins try to use this path, and throw warnings if they cannot access it. -cmd_prefix mkdir /.pytest_cache -cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh new file mode 100755 index 000000000..9eb0def9f --- /dev/null +++ b/.github/workflows/scripts/build_python_client.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -python-client.tar and -python-client-docs.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +rm -rf "pulp_python-client" + +./gen-client.sh "../pulp_python/python-api.json" "python" python "pulp_python" + +pushd pulp_python-client +python -m build + +twine check "dist/pulp_python_client-"*"-py3-none-any.whl" +twine check "dist/pulp_python_client-"*".tar.gz" + +tar cvf "../../pulp_python/python-python-client.tar" ./dist + +find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; +find ./docs/* -exec sed -i 's/README//g' {} \; +cp README.md docs/index.md +sed -i 's/docs\///g' docs/index.md +find ./docs/* -exec sed -i 's/\.md//g' {} \; + +cat >> mkdocs.yml << DOCSYAML +--- +site_name: PulpPython Client +site_description: Python bindings +site_author: Pulp Team +site_url: https://docs.pulpproject.org/pulp_python_client/ +repo_name: pulp/pulp_python +repo_url: https://github.com/pulp/pulp_python +theme: readthedocs +DOCSYAML + +# Building the bindings docs +mkdocs build + +# Pack the built site. +tar cvf ../../pulp_python/python-python-client-docs.tar ./site +popd +popd diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh new file mode 100755 index 000000000..6b433edd2 --- /dev/null +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -ruby-client.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +rm -rf "pulp_python-client" + +./gen-client.sh "../pulp_python/python-api.json" "python" ruby "pulp_python" + +pushd pulp_python-client +gem build pulp_python_client +tar cvf "../../pulp_python/python-ruby-client.tar" "./pulp_python_client-"*".gem" +popd +popd diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index 7780e8be0..400e05423 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -8,22 +8,11 @@ # For more info visit https://github.com/pulp/plugin_template # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." set -euv -echo ::group::REQUESTS -pip3 install requests - -pip3 install pygithub - -echo ::endgroup:: - -for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') +for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') do - python3 .ci/scripts/validate_commit_message.py $sha - VALUE=$? - if [ "$VALUE" -gt 0 ]; then - exit $VALUE - fi + python3 .ci/scripts/validate_commit_message.py "$SHA" done diff --git a/.github/workflows/scripts/create_release_from_tag.sh b/.github/workflows/scripts/create_release_from_tag.sh deleted file mode 100755 index 328454e0d..000000000 --- a/.github/workflows/scripts/create_release_from_tag.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -e - -curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/releases \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "tag_name": "$1", - "name": "$1" -} -EOF diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py deleted file mode 100755 index 61ae6e478..000000000 --- a/.github/workflows/scripts/docs-publisher.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import subprocess -import os -import re -from shutil import rmtree -import tempfile -import requests -import json -from packaging import version - -WORKING_DIR = os.environ["WORKSPACE"] - -VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" -RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" - -USERNAME = "doc_builder_pulp_python" -HOSTNAME = "8.43.85.236" - -SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_python/" - - -def make_directory_with_rsync(remote_paths_list): - """ - Ensure the remote directory path exists. - - :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the - remote. - :type remote_paths_list: a list of strings, with each string representing a directory. - """ - try: - tempdir_path = tempfile.mkdtemp() - cwd = os.getcwd() - os.chdir(tempdir_path) - os.makedirs(os.sep.join(remote_paths_list)) - remote_path_arg = "%s@%s:%s%s" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - remote_paths_list[0], - ) - local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep - rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command) - if exit_code != 0: - raise RuntimeError("An error occurred while creating remote directories.") - finally: - rmtree(tempdir_path) - os.chdir(cwd) - - -def ensure_dir(target_dir, clean=True): - """ - Ensure that the directory specified exists and is empty. - - By default this will delete the directory if it already exists - - :param target_dir: The directory to process - :type target_dir: str - :param clean: Whether or not the directory should be removed and recreated - :type clean: bool - """ - if clean: - rmtree(target_dir, ignore_errors=True) - try: - os.makedirs(target_dir) - except OSError: - pass - - -def main(): - """ - Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--build-type", required=True, help="Build type: nightly, tag or changelog." - ) - parser.add_argument("--branch", required=True, help="Branch or tag name.") - opts = parser.parse_args() - - build_type = opts.build_type - - branch = opts.branch - - publish_at_root = False - - # rsync the docs - print("rsync the docs") - docs_directory = os.sep.join([WORKING_DIR, "docs"]) - local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep - if build_type == "nightly": - # This is a nightly build - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - make_directory_with_rsync(["en", branch, build_type]) - rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: - # Only publish docs at the root if this is the latest version - r = requests.get("https://pypi.org/pypi/pulp-python/json") - latest_version = version.parse(json.loads(r.text)["info"]["version"]) - docs_version = version.parse(branch) - # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. - # It doesn't prevent incorrect docs being published at root if 2 releases are done close - # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately - # after. - if docs_version >= latest_version: - publish_at_root = True - # Post releases should use the x.y.z part of the version string to form a path - if "post" in branch: - branch = ".".join(branch.split(".")[:-1]) - - # This is a GA build. - # publish to the root of docs.pulpproject.org - if publish_at_root: - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "en", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - # This is a pre-release - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "nightly", - "--exclude", - "testing", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "changelog": - if branch != "main": - raise RuntimeError("Can only publish CHANGELOG from main") - # Publish the CHANGELOG from main branch at the root directory - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - changelog_local_path_arg = os.path.join(local_path_arg, "changes.html") - rsync_command = [ - "rsync", - "-vzh", - "--omit-dir-times", - changelog_local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - raise RuntimeError("Build type must be either 'nightly', 'tag' or 'changelog'.") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 292a3ed50..9df47ccde 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -7,132 +7,43 @@ # # For more info visit https://github.com/pulp/plugin_template +set -euv + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. REPO_ROOT="$PWD" -set -euv - source .github/workflows/scripts/utils.sh -export PULP_API_ROOT="/pulp/" - PIP_REQUIREMENTS=("pulp-cli") -if [[ "$TEST" = "docs" || "$TEST" = "publish" ]] -then - PIP_REQUIREMENTS+=("-r" "doc_requirements.txt") - git clone https://github.com/pulp/pulpcore.git ../pulpcore - PIP_REQUIREMENTS+=("psycopg2-binary" "-r" "../pulpcore/doc_requirements.txt") -fi +# This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} -if [[ "$TEST" != "docs" ]] +if [[ "$TEST" = "s3" ]]; then +for i in {1..3} +do + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 +done +if [[ $s -gt 0 ]] then - PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" - git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -fi - -cd .ci/ansible/ - -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - PLUGIN_NAME=./pulp_python/dist/pulp_python-$PLUGIN_VERSION-py3-none-any.whl -else - PLUGIN_NAME=./pulp_python + echo "Failed to install amazon.aws" + exit $s fi -cat >> vars/main.yaml << VARSYAML -image: - name: pulp - tag: "ci_build" -plugins: - - name: pulp_python - source: "${PLUGIN_NAME}" -VARSYAML -if [[ -f ../../ci_requirements.txt ]]; then - cat >> vars/main.yaml << VARSYAML - ci_requirements: true -VARSYAML fi -cat >> vars/main.yaml << VARSYAML -services: - - name: pulp - image: "pulp:ci_build" - volumes: - - ./settings:/etc/pulp - - ./ssh:/keys/ - - ~/.config:/var/lib/pulp/.config - - ../../../pulp-openapi-generator:/root/pulp-openapi-generator - env: - PULP_WORKERS: "4" - PULP_HTTPS: "true" -VARSYAML - -cat >> vars/main.yaml << VARSYAML -pulp_env: {} -pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} -pulp_scheme: https - -pulp_container_tag: "latest" - -VARSYAML - -if [ "$TEST" = "s3" ]; then - export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA - export MINIO_SECRET_KEY=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 - sed -i -e '/^services:/a \ - - name: minio\ - image: minio/minio\ - env:\ - MINIO_ACCESS_KEY: "'$MINIO_ACCESS_KEY'"\ - MINIO_SECRET_KEY: "'$MINIO_SECRET_KEY'"\ - command: "server /data"' vars/main.yaml - sed -i -e '$a s3_test: true\ -minio_access_key: "'$MINIO_ACCESS_KEY'"\ -minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: null\ -pulp_scenario_env: {}\ -' vars/main.yaml - export PULP_API_ROOT="/rerouted/djnd/" -fi - -if [ "$TEST" = "azure" ]; then - mkdir -p azurite - cd azurite - openssl req -newkey rsa:2048 -x509 -nodes -keyout azkey.pem -new -out azcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-azurite" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN" - sudo cp azcert.pem /usr/local/share/ca-certificates/azcert.crt - sudo dpkg-reconfigure ca-certificates - cd .. - sed -i -e '/^services:/a \ - - name: ci-azurite\ - image: mcr.microsoft.com/azure-storage/azurite\ - volumes:\ - - ./azurite:/etc/pulp\ - command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml - sed -i -e '$a azure_test: true\ -pulp_scenario_settings: null\ -pulp_scenario_env: {}\ -' vars/main.yaml -fi +# Check out the pulp-cli branch matching the installed version. +PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" +git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" +cd .ci/ansible/ -if [ "${PULP_API_ROOT:-}" ]; then - sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml -fi - -pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" -if [[ "$TEST" != "docs" ]]; then - cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" -fi +pulp config create --base-url https://pulp --api-root "${PULP_API_ROOT}" --username "admin" --password "password" +cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml ansible-playbook start_container.yaml -# .config needs to be accessible by the pulp user in the container, but some -# files will likely be modified on the host by post/pre scripts. -chmod 777 ~/.config/pulp_smash/ -chmod 666 ~/.config/pulp_smash/settings.json # Plugins often write to ~/.config/pulp/cli.toml from the host chmod 777 ~/.config/pulp chmod 666 ~/.config/pulp/cli.toml @@ -144,27 +55,24 @@ sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certi # Hack: adding pulp CA to certifi.where() CERTIFI=$(python -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERTIFI" > /dev/null -if [[ "$TEST" = "azure" ]]; then - cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a "$CERTIFI" > /dev/null -fi # Hack: adding pulp CA to default CA file CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)') -cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null +cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERT" > /dev/null # Updating certs sudo update-ca-certificates echo ::endgroup:: if [[ "$TEST" = "azure" ]]; then - AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())') - cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /usr/local/lib/python3.8/site-packages/certifi/cacert.pem > /dev/null - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /etc/pki/tls/cert.pem > /dev/null - AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://ci-azurite:10000/devstoreaccount1;' + AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING fi -echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" -echo ::endgroup:: +# Needed for some functional tests +cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" +cmd_prefix bash -c "usermod -a -G wheel pulp" + +# Lots of plugins try to use this path, and throw warnings if they cannot access it. +cmd_prefix mkdir /.pytest_cache +cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh deleted file mode 100755 index 553c78cdc..000000000 --- a/.github/workflows/scripts/install_python_client.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -export PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -pip install twine wheel - -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) - -if [ "$response" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Installing from PyPI." - docker exec pulp pip3 install pulp-python-client==$VERSION - mkdir -p dist - tar cvf python-client.tar ./dist - exit -fi - -cd ../pulp-openapi-generator -rm -rf pulp_python-client -./generate.sh pulp_python python $VERSION -cd pulp_python-client -python setup.py sdist bdist_wheel --python-tag py3 -find . -name "*.whl" -exec docker exec pulp pip3 install /root/pulp-openapi-generator/pulp_python-client/{} \; -tar cvf ../../pulp_python/python-client.tar ./dist - -find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; -find ./docs/* -exec sed -i 's/README//g' {} \; -cp README.md docs/index.md -sed -i 's/docs\///g' docs/index.md -find ./docs/* -exec sed -i 's/\.md//g' {} \; -tar cvf ../../pulp_python/python-client-docs.tar ./docs -exit $? diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh deleted file mode 100755 index 1dd84089a..000000000 --- a/.github/workflows/scripts/install_ruby_client.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -export PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" - -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) - -if [ "$response" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Installing from RubyGems.org." - gem install pulp_python_client -v $VERSION - touch pulp_python_client-$VERSION.gem - tar cvf ruby-client.tar ./pulp_python_client-$VERSION.gem - exit -fi - -cd ../pulp-openapi-generator -rm -rf pulp_python-client -./generate.sh pulp_python ruby $VERSION -cd pulp_python-client -gem build pulp_python_client -gem install --both ./pulp_python_client-$VERSION.gem -tar cvf ../../pulp_python/ruby-client.tar ./pulp_python_client-$VERSION.gem diff --git a/.github/workflows/scripts/post_docs_test.sh b/.github/workflows/scripts/post_docs_test.sh deleted file mode 100644 index 2178e5a84..000000000 --- a/.github/workflows/scripts/post_docs_test.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env sh - -export BASE_ADDR=https://pulp -export CONTENT_ADDR=https://pulp - -pip install -r doc_requirements.txt - -cd docs/_scripts/ -bash ./quickstart.sh diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh deleted file mode 100755 index 2cd1d0523..000000000 --- a/.github/workflows/scripts/publish_client_gem.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - - -mkdir ~/.gem || true -touch ~/.gem/credentials -echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials -sudo chmod 600 ~/.gem/credentials - -export VERSION=$(ls pulp_python_client* | sed -rn 's/pulp_python_client-(.*)\.gem/\1/p') - -if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) - -if [ "$response" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Skipping." - exit -fi - -GEM_FILE="$(ls pulp_python_client-*)" -gem push ${GEM_FILE} diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh deleted file mode 100755 index 83dbcaf60..000000000 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -pip install twine - -export VERSION=$(ls dist | sed -rn 's/pulp_python-client-(.*)\.tar.gz/\1/p') - -if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) - -if [ "$response" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Skipping." - exit -fi - -twine check dist/pulp_python_client-$VERSION-py3-none-any.whl || exit 1 -twine check dist/pulp_python-client-$VERSION.tar.gz || exit 1 -twine upload dist/pulp_python_client-$VERSION-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/pulp_python-client-$VERSION.tar.gz -u pulp -p $PYPI_PASSWORD - -exit $? diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh deleted file mode 100755 index b82378f24..000000000 --- a/.github/workflows/scripts/publish_docs.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -mkdir ~/.ssh -touch ~/.ssh/pulp-infra -chmod 600 ~/.ssh/pulp-infra -echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra - -echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts -chmod 644 ~/.ssh/known_hosts - -pip3 install packaging - -export PYTHONUNBUFFERED=1 -export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export WORKSPACE=$PWD - -eval "$(ssh-agent -s)" #start the ssh agent -ssh-add ~/.ssh/pulp-infra - -python3 .github/workflows/scripts/docs-publisher.py --build-type $1 --branch $2 - -if [[ "$GITHUB_WORKFLOW" == "Python changelog update" ]]; then - # Do not build bindings docs on changelog update - exit -fi - -pip install mkdocs pymdown-extensions "Jinja2<3.1" - -mkdir -p ../bindings -tar -xvf python-client-docs.tar --directory ../bindings -cd ../bindings -cat >> mkdocs.yml << DOCSYAML ---- -site_name: PulpPython Client -site_description: Python bindings -site_author: Pulp Team -site_url: https://docs.pulpproject.org/pulp_python_client/ -repo_name: pulp/pulp_python -repo_url: https://github.com/pulp/pulp_python -theme: readthedocs -DOCSYAML - -# Building the bindings docs -mkdocs build - -# publish to docs.pulpproject.org/pulp_python_client -rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/ - -# publish to docs.pulpproject.org/pulp_python_client/en/{release} -rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/en/"$2" diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh deleted file mode 100755 index 981f77073..000000000 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -set -euv - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$1/) -if [ "$response" == "200" ]; -then - echo "pulp_python $1 has already been released. Skipping." - exit -fi - -pip install twine - -twine check dist/pulp_python-$1-py3-none-any.whl || exit 1 -twine check dist/pulp-python-$1.tar.gz || exit 1 -twine upload dist/pulp_python-$1-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/pulp-python-$1.tar.gz -u pulp -p $PYPI_PASSWORD - -exit $? diff --git a/.github/workflows/scripts/push_branch_and_tag_to_github.sh b/.github/workflows/scripts/push_branch_and_tag_to_github.sh index 2f13c78d8..421eb1713 100755 --- a/.github/workflows/scripts/push_branch_and_tag_to_github.sh +++ b/.github/workflows/scripts/push_branch_and_tag_to_github.sh @@ -1,21 +1,16 @@ #!/bin/sh -set -e -BRANCH_NAME=$(echo $GITHUB_REF | sed -rn 's/refs\/heads\/(.*)/\1/p') +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template -ref_string=$(git show-ref --tags | grep refs/tags/$1) +set -eu -SHA=${ref_string:0:40} +BRANCH_NAME="$(echo "$GITHUB_REF" | sed -rn 's/refs\/heads\/(.*)/\1/p')" -remote_repo=https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git +remote_repo="https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" -git push "${remote_repo}" $BRANCH_NAME - -curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/git/refs \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "ref": "refs/tags/$1", - "sha": "$SHA" -} -EOF +git push "${remote_repo}" "$BRANCH_NAME" "$1" diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py deleted file mode 100755 index 04d2e0d90..000000000 --- a/.github/workflows/scripts/release.py +++ /dev/null @@ -1,184 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import re -import os -import textwrap -import requests -import subprocess - -from git import Repo -from pathlib import Path - - -def get_package_from_pypi(version, plugin_path): - """ - Download a package from PyPI. - - :param version: version of the package to download from PyPI - :return: True/False if download was successful - """ - os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) - r = requests.get(f"https://pypi.org/pypi/pulp-python/{version}/json") - if r.status_code == 200: - metadata = r.json() - for url_data in metadata["urls"]: - filename = url_data["filename"] - r2 = requests.get(url_data["url"]) - if r2.status_code != 200: - raise RuntimeError(f"Failed to download released artifact {filename}") - with open(os.path.join(plugin_path, "dist", filename), "wb") as f: - f.write(r2.content) - return True - return False - - -def create_release_commits(repo, release_version, plugin_path): - """Build changelog, set version, commit, bump to next dev version, commit.""" - issues_to_close = set() - for filename in Path(f"{plugin_path}/CHANGES").rglob("*"): - if filename.stem.isdigit(): - issue = filename.stem - issues_to_close.add(issue) - - issues = ",".join(issues_to_close) - # First commit: changelog - os.system(f"towncrier --yes --version {release_version}") - git = repo.git - git.add("CHANGES.rst") - git.add("CHANGES/*") - git.commit("-m", f"Add changelog for {release_version}\n\n[noissue]") - - # Second commit: release version - os.system("bump2version release --allow-dirty") - - git.add(f"{plugin_path}/pulp_python/*") - git.add(f"{plugin_path}/docs/conf.py") - git.add(f"{plugin_path}/setup.py") - git.add(f"{plugin_path}/requirements.txt") - git.add(f"{plugin_path}/.bumpversion.cfg") - git.commit("-m", f"Release {release_version}\nGH Issues: {issues}\n\n[noissue]") - sha = repo.head.object.hexsha - short_sha = git.rev_parse(sha, short=7) - - os.system("bump2version patch --allow-dirty") - - new_dev_version = None - with open(f"{plugin_path}/setup.py") as fp: - for line in fp.readlines(): - if "version=" in line: - new_dev_version = re.split("\"|'", line)[1] - if not new_dev_version: - raise RuntimeError("Could not detect new dev version ... aborting.") - - git.add(f"{plugin_path}/pulp_python/*") - git.add(f"{plugin_path}/docs/conf.py") - git.add(f"{plugin_path}/setup.py") - git.add(f"{plugin_path}/requirements.txt") - git.add(f"{plugin_path}/.bumpversion.cfg") - git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]") - print(f"Release commit == {short_sha}") - print(f"All changes were committed on branch: release_{release_version}") - return sha - - -def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): - """Create a tag if one is needed and build a package if one is not on PyPI.""" - # Remove auth header config - with repo.config_writer() as conf: - conf.remove_section('http "https://github.com/"') - conf.release() - - # Determine if a tag exists and if it matches the specified commit sha - tag = None - for existing_tag in repo.tags: - if existing_tag.name == desired_tag: - if existing_tag.commit.hexsha == commit_sha: - tag = existing_tag - else: - raise RuntimeError( - f"The '{desired_tag}' tag already exists, but the commit sha does not match " - f"'{commit_sha}'." - ) - - # Create a tag if one does not exist - if not tag: - tag = repo.create_tag(desired_tag, ref=commit_sha) - - # Checkout the desired tag and reset the tree - repo.head.reference = tag.commit - repo.head.reset(index=True, working_tree=True) - - # Check if Package is available on PyPI - if not get_package_from_pypi(tag.name, plugin_path): - os.system("python3 setup.py sdist bdist_wheel --python-tag py3") - - -def main(): - helper = textwrap.dedent( - """\ - Start the release process. - - Example: - setup.py on plugin before script: - version="2.0.0.dev" - - $ python .ci/scripts/release.py - - setup.py on plugin after script: - version="2.0.1.dev" - - - """ - ) - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, description=helper - ) - - parser.add_argument( - "release_version", - type=str, - help="The version string for the release.", - ) - - args = parser.parse_args() - - release_version_arg = args.release_version - - release_path = os.path.dirname(os.path.abspath(__file__)) - plugin_path = release_path.split("/.github")[0] - - output = subprocess.check_output(["bump2version", "--dry-run", "--list", "release"]) - release_version = re.findall(r"\nnew_version=([0-9.]*)\n", output.decode())[0] - - print(f"\n\nRepo path: {plugin_path}") - repo = Repo(plugin_path) - - release_commit = None - if release_version != release_version_arg: - # Look for a commit with the requested release version - for commit in repo.iter_commits(): - if f"Release {release_version_arg}\n" in commit.message: - release_commit = commit - release_version = release_version_arg - break - if not release_commit: - raise RuntimeError( - f"The release version {release_version_arg} does not match the .dev version at " - "HEAD. A release commit for such version does not exist." - ) - - if not release_commit: - release_commit_sha = create_release_commits(repo, release_version, plugin_path) - else: - release_commit_sha = release_commit.hexsha - create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh new file mode 100755 index 000000000..a08353cdb --- /dev/null +++ b/.github/workflows/scripts/release.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -eu -o pipefail + +BRANCH=$(git branch --show-current) + +if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]] +then + echo ERROR: This is not a release branch! + exit 1 +fi + +# The tail is a necessary workaround to remove the warning from the output. +NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)" +echo "Release ${NEW_VERSION}" + +if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] +then + echo ERROR: Version does not match release branch + exit 1 +fi + +towncrier build --yes --version "${NEW_VERSION}" +bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump-my-version bump patch --commit + +git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 42ae3d1f2..5a4910e30 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -12,15 +12,13 @@ set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -REPO_ROOT="$PWD" source .github/workflows/scripts/utils.sh export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh -export POST_DOCS_TEST=$PWD/.github/workflows/scripts/post_docs_test.sh export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh -# Needed for both starting the service and building the docs. +# Needed for starting the service # Gets set in .github/settings.yml, but doesn't seem to inherited by # this script. export DJANGO_SETTINGS_MODULE=pulpcore.app.settings @@ -28,114 +26,127 @@ export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py export PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" -if [[ "$TEST" = "docs" ]]; then - if [[ "$GITHUB_WORKFLOW" == "Python CI" ]]; then - pip install towncrier==19.9.0 - towncrier --yes --version 4.0.0.ci - fi - cd docs - make PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24PULP_URL" diagrams html - tar -cvf docs.tar ./_build - cd .. - - if [ -f $POST_DOCS_TEST ]; then - source $POST_DOCS_TEST - fi - exit -fi - REPORTED_STATUS="$(pulp status)" -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - REPORTED_VERSION="$(echo $REPORTED_STATUS | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')" - response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$REPORTED_VERSION/) - if [ "$response" == "200" ]; - then - echo "pulp_python $REPORTED_VERSION has already been released. Skipping running tests." - exit - fi -fi +echo "${REPORTED_STATUS}" echo "machine pulp login admin password password " | cmd_user_stdin_prefix bash -c "cat >> ~pulp/.netrc" # Some commands like ansible-galaxy specifically require 600 -cmd_user_stdin_prefix bash -c "chmod 600 ~pulp/.netrc" +cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -cd ../pulp-openapi-generator -if [ "$(echo "$REPORTED_STATUS" | jq -r '.versions[0].package')" = "null" ] -then - # We are on an old version of pulpcore without package in the status report - for app_label in $(echo "$REPORTED_STATUS" | jq -r '.versions[].component') +# Generate bindings +################### + +echo "::group::Generate bindings" + +touch bindings_requirements.txt +pushd ../pulp-openapi-generator + # Use app_label to generate api.json and package to produce the proper package name. + + # Workaround: Domains are not supported by the published bindings. + # Sadly: Different pulpcore-versions aren't either... + # * In the 'pulp' scenario we use the published/prebuilt bindings, so we can test it. + # * In other scenarios we generate new bindings from server spec, so we have a more + # reliable client. + if [ "$TEST" = "pulp" ] + then + BUILT_CLIENTS=" python " + else + BUILT_CLIENTS="" + fi + + for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") do - if [ "$app_label" = "core" ] + COMPONENT="$(jq -r '.component' <<<"${ITEM}")" + VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")" + # On older status endpoints, the module was not provided, but the package should be accurate + # there, because we did not merge plugins into pulpcore back then. + MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")" + PACKAGE="${MODULE%%.*}" + cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json" + if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]] then - item=pulpcore + rm -rf "./${PACKAGE}-client" + ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" + pushd "${PACKAGE}-client" + python -m build + popd else - item="pulp_${app_label}" + if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] + then + ls -lR "${PACKAGE}-client/" + echo "Error: Client bindings for ${COMPONENT} not found." + echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing." + exit 1 + fi fi - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done -else - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[].package|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" + echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp_python/bindings_requirements.txt" done -fi +popd + +echo "::endgroup::" -cd $REPO_ROOT +echo "::group::Debug bindings diffs" -cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" -cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt +# Bindings diff for python +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "python-api.json" > "build-api.json" +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "../pulp-openapi-generator/python-api.json" > "test-api.json" +jsondiff --indent 2 build-api.json test-api.json || true +echo "::endgroup::" + +# Install test requirements +########################### + +# Carry on previous constraints (there might be no such file). +cat *_constraints.txt > bindings_constraints.txt || true +cat .ci/assets/ci_constraints.txt >> bindings_constraints.txt +# Add a safeguard to make sure the proper versions of the clients are installed. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' >> bindings_constraints.txt +cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') -cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt | tee -a "$CERTIFI" > /dev/null" +cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" # check for any uncommitted migrations echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations python --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs pulp_python.tests.unit" - +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulp_python.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_python.tests.performance" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_python.tests.performance" else - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_python.tests.performance.test_$PERFORMANCE_TEST" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_python.tests.performance.test_${PERFORMANCE_TEST}" fi exit fi -if [ -f $FUNC_TEST_SCRIPT ]; then - source $FUNC_TEST_SCRIPT +if [ -f "$FUNC_TEST_SCRIPT" ]; then + source "$FUNC_TEST_SCRIPT" else - - if [[ "$GITHUB_WORKFLOW" == "Python Nightly CI/CD" ]] || [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" - - - else - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_python.tests.functional -m 'not parallel'" - - - fi - + if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] + then + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" + else + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel'" + fi fi pushd ../pulp-cli pip install -r test_requirements.txt -pytest -v -m pulp_python +pytest -v tests -m "pulp_python" popd -if [ -f $POST_SCRIPT ]; then - source $POST_SCRIPT +if [ -f "$POST_SCRIPT" ]; then + source "$POST_SCRIPT" fi diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py index a6e241165..967984a42 100755 --- a/.github/workflows/scripts/update_backport_labels.py +++ b/.github/workflows/scripts/update_backport_labels.py @@ -32,10 +32,14 @@ def random_color(): assert response.status_code == 200 old_labels = set([x["name"] for x in response.json() if x["name"].startswith("backport-")]) -# get ci_update_branches from template_config.yml +# get list of branches from template_config.yml with open("./template_config.yml", "r") as f: plugin_template = yaml.safe_load(f) -new_labels = set(["backport-" + x for x in plugin_template["ci_update_branches"]]) +branches = set(plugin_template["supported_release_branches"]) +latest_release_branch = plugin_template["latest_release_branch"] +if latest_release_branch is not None: + branches.add(latest_release_branch) +new_labels = {"backport-" + x for x in branches} # delete old labels that are not in new labels for label in old_labels.difference(new_labels): diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh deleted file mode 100755 index b22001b4f..000000000 --- a/.github/workflows/scripts/update_ci.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -eu - -if [ ! -f "template_config.yml" ]; then - echo "No template_config.yml detected." - exit 1 -fi - -pushd ../plugin_template -./plugin-template --github pulp_python -popd - -# Check if only gitref file has changed, so no effect on CI workflows. -if [[ $(git diff --name-only) == ".github/template_gitref" ]]; then - echo "No changes detected in github section." - git restore ".github/template_gitref" -fi - -if [[ $(git status --porcelain) ]]; then - git add -A - git commit -m "Update CI files" -m "[noissue]" -else - echo "No updates needed" -fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..fbddb788b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,144 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Test" +on: + workflow_call: + inputs: + matrix_env: + required: true + type: "string" + +defaults: + run: + working-directory: "pulp_python" + +jobs: + test: + runs-on: "ubuntu-latest" + strategy: + fail-fast: false + matrix: + env: "${{ fromJSON(inputs.matrix_env) }}" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Download plugin package" + uses: "actions/download-artifact@v8" + with: + name: "plugin_package" + path: "pulp_python/dist/" + + - name: "Download API specs" + uses: "actions/download-artifact@v8" + with: + name: "api_spec" + path: "pulp_python/" + + - name: "Download client packages" + uses: "actions/download-artifact@v8" + with: + name: "python-client.tar" + path: "pulp_python" + + - name: "Unpack client packages" + working-directory: "pulp-openapi-generator" + run: | + mkdir -p "pulp_python-client" + pushd "pulp_python-client" + tar xvf "../../pulp_python/python-python-client.tar" + popd + + - name: "Install python dependencies" + run: | + pip install build towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV + + - name: "Set environment variables" + run: | + echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + + - name: "Prepare Scenario Definition" + run: | + .github/workflows/scripts/before_install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Install" + run: | + .github/workflows/scripts/install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Dump CI Metadata" + run: | + .github/workflows/scripts/before_script.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Script" + run: | + .github/workflows/scripts/script.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Extract Deprecations from Logs" + run: | + docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt + + - name: "Upload Deprecations" + uses: "actions/upload-artifact@v5" + with: + name: "deprecations-${{ matrix.env.TEST }}" + path: "pulp_python/deprecations-${{ matrix.env.TEST }}.txt" + if-no-files-found: "error" + retention-days: 5 + overwrite: true + + - name: "Logs" + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list" || true +... diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index 1dd998a2d..39320dcae 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -7,33 +7,32 @@ --- -name: Python Update Labels +name: "Python Update Labels" on: push: branches: - - main + - "main" paths: - - 'template_config.yml' + - "template_config.yml" jobs: update_backport_labels: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v6" with: - python-version: "3.8" - - name: Configure Git with pulpbot name and email + python-version: "3.11" + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Install python dependencies + - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install requests pyyaml - echo ::endgroup:: - - uses: actions/checkout@v3 - - name: Update labels + - uses: "actions/checkout@v6" + - name: "Update labels" run: | python3 .github/workflows/scripts/update_backport_labels.py env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" +... diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 91560ce3d..0c6c2f8f0 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -7,67 +7,69 @@ --- -name: Python CI Update +name: "Python CI Update" on: schedule: # * is a special character in YAML so you have to quote this string # runs at 2:30 UTC every Sunday - - cron: '30 2 * * 0' - + - cron: "30 2 * * 0" workflow_dispatch: jobs: update: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: "actions/checkout@v6" with: - repository: pulp/plugin_template - path: plugin_template fetch-depth: 0 + repository: "pulp/plugin_template" + path: "plugin_template" - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v6" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | - echo ::group::PYDEPS - pip install gitpython requests packaging jinja2 pyyaml - echo ::endgroup:: + pip install gitpython packaging -r plugin_template/requirements.txt - - name: Configure Git with pulpbot name and email + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - uses: actions/checkout@v3 + - uses: "actions/checkout@v6" with: - path: pulp_python - ref: 'main' fetch-depth: 0 + path: "pulp_python" + ref: "main" - - name: Run update - working-directory: pulp_python + - name: "Run update" + working-directory: "pulp_python" run: | ../plugin_template/scripts/update_ci.sh - - name: Create Pull Request for CI files - uses: peter-evans/create-pull-request@v4 + - name: "Create Pull Request for CI files" + uses: "peter-evans/create-pull-request@v8" + id: "create_pr_main" with: - token: ${{ secrets.RELEASE_TOKEN }} - path: pulp_python - committer: pulpbot - author: pulpbot - title: 'Update CI files for branch main' - body: '[noissue]' - branch: 'update-ci/main' - base: 'main' - commit-message: | - Update CI files - - [noissue] + token: "${{ secrets.RELEASE_TOKEN }}" + path: "pulp_python" + committer: "pulpbot " + author: "pulpbot " + title: "Update CI files for branch main" + branch: "update-ci/main" + base: "main" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_python" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_main.outputs.pull-request-number }}" + if: "steps.create_pr_main.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true +... diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 000000000..8190f38b7 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,489 @@ +# Changelog + +[//]: # (You should *NOT* be adding new change log entries to this file, this) +[//]: # (file is managed by towncrier. You *may* edit previous change logs to) +[//]: # (fix problems like typo corrections or such.) +[//]: # (To add a new change log entry, please see the contributing docs.) +[//]: # (WARNING: Don't drop the towncrier directive!) + +[//]: # (towncrier release notes start) + +## 3.11.7 (2025-11-18) {: #3.11.7 } + +No significant changes. + +--- + +## 3.11.6 (2025-07-23) {: #3.11.6 } + +No significant changes. + +--- + +## 3.11.5 (2025-04-15) {: #3.11.5 } + +No significant changes. + +--- + +## 3.11.4 (2025-02-20) {: #3.11.4 } + +#### Bugfixes {: #3.11.4-bugfix } + +- Fixed the JSONField specification so it doesn't break ruby bindings. + See context [here](https://github.com/pulp/pulp_rpm/issues/3639). + +#### Misc {: #3.11.4-misc } + +- + +--- + +# ## 3.11.3 (2024-08-21) {: #3.11.3 } + +#### Bugfixes {: #3.11.3-bugfix } + +- Fixed uploads not supporting packages using metadata spec 2.3 + [#682](https://github.com/pulp/pulp_python/issues/682) +- Fixed package name normalization issue preventing syncing packages with "." or "_" in their names. + [#716](https://github.com/pulp/pulp_python/issues/716) + +--- + +## 3.11.2 (2024-06-27) {: #3.11.2 } + + +#### Bugfixes {: #3.11.2-bugfix } + +- Fixed the `package_types` filter breaking other remote filters. + [#691](https://github.com/pulp/pulp_python/issues/691) + +--- + +## 3.11.1 (2024-04-11) {: #3.11.1 } + +### Bugfixes + +- Fixed tls_validation not being disabled when set to false on the remote. + [#653](https://github.com/pulp/pulp_python/issues/653) + +--- + +## 3.11.0 (2023-11-08) {: #3.11.0 } + +### Features + +- Added pulpcore 3.40 compatibility. +- Added import export support of python content. + [#579](https://github.com/pulp/pulp_python/issues/579) + +--- + +## 3.10.0 (2023-05-17) {: #3.10.0 } + +### Features + +- Added compatibility for pulpcore 3.25, pulpcore support is now >=3.25,<3.40. + [#605](https://github.com/pulp/pulp_python/issues/605) + +--- + +## 3.9.0 (2023-03-17) {: #3.9.0 } + +### Features + +- Added version filter to package list endpoint. + [#577](https://github.com/pulp/pulp_python/issues/577) +- Allow duplicate uploads to return existing packages instead of erring. + [#590](https://github.com/pulp/pulp_python/issues/590) + +### Bugfixes + +- Fixed pull-through caching ignoring remote proxy settings. + [#553](https://github.com/pulp/pulp_python/issues/553) +- Changed includes and excludes openapi schema to report as array of strings instead of object. + [#576](https://github.com/pulp/pulp_python/issues/576) +- Fixed syncing ignoring remote proxy. + [#581](https://github.com/pulp/pulp_python/issues/581) +- Fixed duplicate operationID for generated PyPI simple endpoints schema. + [#594](https://github.com/pulp/pulp_python/issues/594) + +--- + +## 3.8.0 (2022-12-19) {: #3.8.0 } + +### Bugfixes + +- Fixed syncing failing when using bandersnatch 5.3.0 + [#554](https://github.com/pulp/pulp_python/issues/554) +- Prevent .netrc file from being read on syncs. + [#566](https://github.com/pulp/pulp_python/issues/566) +- Fix 500 error when pip installing using object storage. + [#572](https://github.com/pulp/pulp_python/issues/572) + +### Improved Documentation + +- Documented `pulp_python` specific settings. + [#571](https://github.com/pulp/pulp_python/issues/571) + +--- + +## 3.7.3 (2022-10-06) {: #3.7.3 } + +### Bugfixes + +- Prevent .netrc file from being read on syncs. + [#566](https://github.com/pulp/pulp_python/issues/566) + +--- + +## 3.7.2 (2022-08-04) {: #3.7.2 } + +### Bugfixes + +- Fixed syncing failing when using bandersnatch 5.3.0 + [#554](https://github.com/pulp/pulp_python/issues/554) + +--- + +## 3.7.1 (2022-06-29) {: #3.7.1 } + +No significant changes. + +--- + +## 3.7.0 (2022-06-22) {: #3.7.0 } + +### Features + +- Added ability to fully sync repositories that don't support the PyPI XMLRPC endpoints. Full Pulp-to-Pulp syncing is now available. + [#462](https://github.com/pulp/pulp_python/issues/462) + +### Bugfixes + +- Ensured temporary package uploads are written to worker's directory instead of /tmp. + [#505](https://github.com/pulp/pulp_python/issues/505) + +### Misc + +- [#503](https://github.com/pulp/pulp_python/issues/503) + +--- + +## 3.6.1 (2022-08-19) {: #3.6.1 } + +### Bugfixes + +- Fixed syncing failing when using bandersnatch 5.3.0 + [#554](https://github.com/pulp/pulp_python/issues/554) + +--- + +## 3.6.0 (2021-12-15) {: #3.6.0 } + +### Features + +- `pulp_python` now supports pull-through caching. Add a remote to a distribution to enable this feature. + [#381](https://github.com/pulp/pulp_python/issues/381) +- Enable Azure support + [#458](https://github.com/pulp/pulp_python/issues/458) + +### Bugfixes + +- Fixed proxy url not being passed during sync + [#433](https://github.com/pulp/pulp_python/issues/433) +- Changed the use of `dispatch` to match the signature from pulpcore>=3.15. + [#443](https://github.com/pulp/pulp_python/issues/443) +- Fixed package name normalization issue preventing installing packages with "." or "_" in their names. + [#467](https://github.com/pulp/pulp_python/issues/467) + +--- + +## 3.5.2 (2021-10-05) {: #3.5.2 } + +### Bugfixes + +- Fixed proxy url not being passed during sync + (backported from #445) + [#436](https://github.com/pulp/pulp_python/issues/436) +- Changed the use of `dispatch` to match the signature from pulpcore>=3.15. + (backported from #443) + [#446](https://github.com/pulp/pulp_python/issues/446) + +--- + +## 3.5.1 (2021-09-10) {: #3.5.1 } + +### Bugfixes + +- Fixed proxy url not being passed during sync + (backported from #433) + [#436](https://github.com/pulp/pulp_python/issues/436) + +--- + +## 3.5.0 (2021-08-30) {: #3.5.0 } + +### Features + +- Python package content can now be filtered by their sha256 + [#404](https://github.com/pulp/pulp_python/issues/404) +- Added new setting `PYPI_API_HOSTNAME` that is used to form a distribution's `base_url`. Defaults to the machine's FQDN. + [#412](https://github.com/pulp/pulp_python/issues/412) +- Enabled reclaim disk feature provided by pulpcore 3.15+. + [#425](https://github.com/pulp/pulp_python/issues/425) + +### Bugfixes + +- Fixed twine upload failing when using remote storage backends + [#400](https://github.com/pulp/pulp_python/issues/400) +- Fixed improper metadata serving when using publications with S3 storage + [#413](https://github.com/pulp/pulp_python/issues/413) + +### Deprecations and Removals + +- Dropped support for Python < 3.8. + [#402](https://github.com/pulp/pulp_python/issues/402) + +### Misc + +- [#408](https://github.com/pulp/pulp_python/issues/408), [#427](https://github.com/pulp/pulp_python/issues/427) + +--- + +## 3.4.1 (2021-08-24) {: #3.4.1 } + +### Features + +- Python package content can now be filtered by their sha256 + (backported from #404) + [#419](https://github.com/pulp/pulp_python/issues/419) + +### Bugfixes + +- Fixed improper metadata serving when using publications with S3 storage + (backported from #413) + [#418](https://github.com/pulp/pulp_python/issues/418) +- Fixed twine upload failing when using remote storage backends + (backported from #400) + [#420](https://github.com/pulp/pulp_python/issues/420) + +--- + +3.4.0 (2021-06-17) + +### Features + +- Added `twine` (and other similar Python tools) package upload support + [#342](https://github.com/pulp/pulp_python/issues/342) +- PyPI endpoints are now available at `/pypi/{base_path}/` + [#376](https://github.com/pulp/pulp_python/issues/376) +- Changed the global uniqueness constraint for `PythonPackageContent` to its sha256 digest + [#380](https://github.com/pulp/pulp_python/issues/380) + +### Bugfixes + +- Added missing fields to PyPI live JSON API to be compliant with core metadata version 2.1 + [#352](https://github.com/pulp/pulp_python/issues/352) +- Fixed sync to use default concurrency (10) when download_concurrency was not specified + [#391](https://github.com/pulp/pulp_python/issues/391) + +--- + +## 3.3.0 (2021-05-27) {: #3.3.0 } + +### Features + +- Add support for automatic publishing and distributing. + [#365](https://github.com/pulp/pulp_python/issues/365) + +### Bugfixes + +- Fixed publications publishing more content than was in the repository + [#362](https://github.com/pulp/pulp_python/issues/362) + +### Improved Documentation + +- Update syntax in doc for cli repository content add command + [#368](https://github.com/pulp/pulp_python/issues/368) + +### Misc + +- [#347](https://github.com/pulp/pulp_python/issues/347), [#360](https://github.com/pulp/pulp_python/issues/360), [#371](https://github.com/pulp/pulp_python/issues/371) + +--- + +## 3.2.0 (2021-04-14) {: #3.2.0 } + +### Features + +- Added new sync filter keep_latest_packages to specify how many latest versions of packages to sync + [#339](https://github.com/pulp/pulp_python/issues/339) +- Added new sync filters package_types and exclude_platforms to specify package types to sync + [#341](https://github.com/pulp/pulp_python/issues/341) + +### Misc + +- [#354](https://github.com/pulp/pulp_python/issues/354) + +--- + +## 3.1.0 (2021-03-12) {: #3.1.0 } + +### Features + +- Python content can now be filtered by requires_python + [#3629](https://pulp.plan.io/issues/3629) + +### Improved Documentation + +- Updated workflows to use Pulp CLI commands + [#8364](https://pulp.plan.io/issues/8364) + +--- + +## 3.0.0 (2021-01-12) {: #3.0.0 } + +### Bugfixes + +- Remote proxy settings are now passed to Bandersnatch while syncing + [#7864](https://pulp.plan.io/issues/7864) + +### Improved Documentation + +- Added bullet list of Python Plugin features and a tech preview page for new experimental features + [#7628](https://pulp.plan.io/issues/7628) + +--- + +## 3.0.0b12 (2020-11-05) + +### Features + +- Pulp Python can now fully mirror all packages from PyPi + [#985](https://pulp.plan.io/issues/985) +- Implemented PyPi's json API at content endpoint '/pypi/{package-name}/json'. Pulp can now perform basic syncing on other Pulp Python instances. + [#2886](https://pulp.plan.io/issues/2886) +- Pulp Python now uses Bandersnatch to perform syncing and filtering of package metadata + [#6930](https://pulp.plan.io/issues/6930) + +### Bugfixes + +- Sync now includes python package's classifiers in the content unit + [#3627](https://pulp.plan.io/issues/3627) +- Policy can now be specified when creating a remote from a Bandersnatch config + [#7331](https://pulp.plan.io/issues/7331) +- Includes/excludes/prereleases fields are now properly set in a remote from Bandersnatch config + [#7392](https://pulp.plan.io/issues/7392) + +### Improved Documentation + +- Fixed makemigrations commands in the install docs + [#5386](https://pulp.plan.io/issues/5386) + +### Misc + +- [#6875](https://pulp.plan.io/issues/6875), [#7401](https://pulp.plan.io/issues/7401) + +--- + +## 3.0.0b11 (2020-08-18) + +Compatibility update for pulpcore 3.6 + +--- + +## 3.0.0b10 (2020-08-05) + +### Features + +- Added a new endpoint to remotes "/from_bandersnatch" that allows for Python remote creation from a Bandersnatch config file. + [#6929](https://pulp.plan.io/issues/6929) + +### Bugfixes + +- Including requirements.txt on MANIFEST.in + [#6891](https://pulp.plan.io/issues/6891) +- Updating API to not return publications that aren't complete. + [#6987](https://pulp.plan.io/issues/6987) +- Fixed an issue that prevented 'on_demand' content from being published. + [#7128](https://pulp.plan.io/issues/7128) + +### Improved Documentation + +- Change the commands for publication and distribution on the publish workflow to use their respective scripts already defined in _scripts. + [#6877](https://pulp.plan.io/issues/6877) +- Updated sync.sh, publication.sh and distribution.sh in docs/_scripts to reference wait_until_task_finished function from base.sh + [#6918](https://pulp.plan.io/issues/6918) + +--- + +## 3.0.0b9 (2020-06-01) + +### Features + +- Add upload functionality to the python contents endpoints. + [#5464](https://pulp.plan.io/issues/5464) + +### Bugfixes + +- Fixed the 500 error returned by the OpenAPI schema endpoint. + [#5452](https://pulp.plan.io/issues/5452) + +### Improved Documentation + +- Change the prefix of Pulp services from pulp-* to pulpcore-* + [#4554](https://pulp.plan.io/issues/4554) +- Added "python/python/" to fix two commands in repo.sh, fixed export command in sync.sh + [#6790](https://pulp.plan.io/issues/6790) +- Added "index.html" to the relative_path field for both project_metadata and index_metadata. Added a "/" to fix the link in the simple_index_template. + [#6792](https://pulp.plan.io/issues/6792) +- Updated the workflow documentation for upload.html. Fixed the workflow commands and added more details to the instructions. + [#6854](https://pulp.plan.io/issues/6854) + +### Deprecations and Removals + +- Change _id, _created, _last_updated, _href to pulp_id, pulp_created, pulp_last_updated, pulp_href + [#5457](https://pulp.plan.io/issues/5457) + +- Remove "_" from _versions_href, _latest_version_href + [#5548](https://pulp.plan.io/issues/5548) + +- Removing base field: _type . + [#5550](https://pulp.plan.io/issues/5550) + +- Sync is no longer available at the {remote_href}/sync/ repository={repo_href} endpoint. Instead, use POST {repo_href}/sync/ remote={remote_href}. + + Creating / listing / editing / deleting python repositories is now performed on /pulp/api/v3/python/python/ instead of /pulp/api/v3/repositories/. Only python content can be present in a python repository, and only a python repository can hold python content. + [#5625](https://pulp.plan.io/issues/5625) + +### Misc + +- [#remotetests](https://pulp.plan.io/issues/remotetests), [#4681](https://pulp.plan.io/issues/4681), [#4682](https://pulp.plan.io/issues/4682), [#5304](https://pulp.plan.io/issues/5304), [#5471](https://pulp.plan.io/issues/5471), [#5580](https://pulp.plan.io/issues/5580), [#5701](https://pulp.plan.io/issues/5701) + +--- + +## 3.0.0b8 (2019-09-16) + +### Misc + +- [#4681](https://pulp.plan.io/issues/4681) + +--- + +## 3.0.0b7 (2019-08-01) + +### Features + +- Users can upload a file to create content and optionally add to a repo in one step known as + one-shot upload + [#4396](https://pulp.plan.io/issues/4396) +- Override the Remote's serializer to allow policy='on_demand' and policy='streamed'. + [#4990](https://pulp.plan.io/issues/4990) + +### Improved Documentation + +- Switch to using [towncrier](https://github.com/hawkowl/towncrier) for better release notes. + [#4875](https://pulp.plan.io/issues/4875) + +--- diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index 33b23e839..000000000 --- a/CHANGES.rst +++ /dev/null @@ -1,589 +0,0 @@ -========= -Changelog -========= - -.. - You should *NOT* be adding new change log entries to this file, this - file is managed by towncrier. You *may* edit previous change logs to - fix problems like typo corrections or such. - To add a new change log entry, please see - https://docs.pulpproject.org/en/3.0/nightly/contributing/git.html#changelog-update - - WARNING: Don't drop the next directive! - -.. towncrier release notes start - -3.10.0 (2023-05-17) -=================== - - -Features --------- - -- Added compatibility for pulpcore 3.25, pulpcore support is now >=3.25,<3.40. - `#605 `__ - - ----- - - -3.9.0 (2023-03-17) -================== - - -Features --------- - -- Added version filter to package list endpoint. - `#577 `__ -- Allow duplicate uploads to return existing packages instead of erring. - `#590 `__ - - -Bugfixes --------- - -- Fixed pull-through caching ignoring remote proxy settings. - `#553 `__ -- Changed includes and excludes openapi schema to report as array of strings instead of object. - `#576 `__ -- Fixed syncing ignoring remote proxy. - `#581 `__ -- Fixed duplicate operationID for generated PyPI simple endpoints schema. - `#594 `__ - - ----- - - -3.8.0 (2022-12-19) -================== - - -Bugfixes --------- - -- Fixed syncing failing when using bandersnatch 5.3.0 - `#554 `__ -- Prevent .netrc file from being read on syncs. - `#566 `__ -- Fix 500 error when pip installing using object storage. - `#572 `__ - - -Improved Documentation ----------------------- - -- Documented ``pulp_python`` specific settings. - `#571 `__ - - ----- - - -3.7.3 (2022-10-06) -================== - - -Bugfixes --------- - -- Prevent .netrc file from being read on syncs. - `#566 `__ - - ----- - - -3.7.2 (2022-08-04) -================== - - -Bugfixes --------- - -- Fixed syncing failing when using bandersnatch 5.3.0 - `#554 `__ - - ----- - - -3.7.1 (2022-06-29) -================== - - -No significant changes. - - ----- - - -3.7.0 (2022-06-22) -================== - - -Features --------- - -- Added ability to fully sync repositories that don't support the PyPI XMLRPC endpoints. Full Pulp-to-Pulp syncing is now available. - `#462 `__ - - -Bugfixes --------- - -- Ensured temporary package uploads are written to worker's directory instead of /tmp. - `#505 `__ - - -Misc ----- - -- `#503 `__ - - ----- - - -3.6.1 (2022-08-19) -================== - - -Bugfixes --------- - -- Fixed syncing failing when using bandersnatch 5.3.0 - `#554 `__ - - ----- - - -3.6.0 (2021-12-15) -================== - - -Features --------- - -- ``pulp_python`` now supports pull-through caching. Add a remote to a distribution to enable this feature. - `#381 `_ -- Enable Azure support - `#458 `_ - - -Bugfixes --------- - -- Fixed proxy url not being passed during sync - `#433 `_ -- Changed the use of ``dispatch`` to match the signature from pulpcore>=3.15. - `#443 `_ -- Fixed package name normalization issue preventing installing packages with "." or "_" in their names. - `#467 `_ - - ----- - - -3.5.2 (2021-10-05) -================== - - -Bugfixes --------- - -- Fixed proxy url not being passed during sync - (backported from #445) - `#436 `_ -- Changed the use of ``dispatch`` to match the signature from pulpcore>=3.15. - (backported from #443) - `#446 `_ - - ----- - - -3.5.1 (2021-09-10) -================== - - -Bugfixes --------- - -- Fixed proxy url not being passed during sync - (backported from #433) - `#436 `_ - - ----- - - -3.5.0 (2021-08-30) -================== - - -Features --------- - -- Python package content can now be filtered by their sha256 - `#404 `_ -- Added new setting ``PYPI_API_HOSTNAME`` that is used to form a distribution's ``base_url``. Defaults to the machine's FQDN. - `#412 `_ -- Enabled reclaim disk feature provided by pulpcore 3.15+. - `#425 `_ - - -Bugfixes --------- - -- Fixed twine upload failing when using remote storage backends - `#400 `_ -- Fixed improper metadata serving when using publications with S3 storage - `#413 `_ - - -Deprecations and Removals -------------------------- - -- Dropped support for Python < 3.8. - `#402 `_ - - -Misc ----- - -- `#408 `_, `#427 `_ - - ----- - - -3.4.1 (2021-08-24) -================== - - -Features --------- - -- Python package content can now be filtered by their sha256 - (backported from #404) - `#419 `_ - - -Bugfixes --------- - -- Fixed improper metadata serving when using publications with S3 storage - (backported from #413) - `#418 `_ -- Fixed twine upload failing when using remote storage backends - (backported from #400) - `#420 `_ - - ----- - - -3.4.0 (2021-06-17) - -Features --------- - -- Added ``twine`` (and other similar Python tools) package upload support - `#342 `_ -- PyPI endpoints are now available at ``/pypi/{base_path}/`` - `#376 `_ -- Changed the global uniqueness constraint for ``PythonPackageContent`` to its sha256 digest - `#380 `_ - - -Bugfixes --------- - -- Added missing fields to PyPI live JSON API to be compliant with core metadata version 2.1 - `#352 `_ -- Fixed sync to use default concurrency (10) when download_concurrency was not specified - `#391 `_ - - ----- - - -3.3.0 (2021-05-27) -================== - - -Features --------- - -- Add support for automatic publishing and distributing. - `#365 `_ - - -Bugfixes --------- - -- Fixed publications publishing more content than was in the repository - `#362 `_ - - -Improved Documentation ----------------------- - -- Update syntax in doc for cli repository content add command - `#368 `_ - - -Misc ----- - -- `#347 `_, `#360 `_, `#371 `_ - - ----- - - -3.2.0 (2021-04-14) -================== - - -Features --------- - -- Added new sync filter `keep_latest_packages` to specify how many latest versions of packages to sync - `#339 `_ -- Added new sync filters `package_types` and `exclude_platforms` to specify package types to sync - `#341 `_ - - -Misc ----- - -- `#354 `_ - - ----- - - -3.1.0 (2021-03-12) -================== - - -Features --------- - -- Python content can now be filtered by requires_python - `#3629 `_ - - -Improved Documentation ----------------------- - -- Updated workflows to use Pulp CLI commands - `#8364 `_ - - ----- - - -3.0.0 (2021-01-12) -================== - - -Bugfixes --------- - -- Remote proxy settings are now passed to Bandersnatch while syncing - `#7864 `_ - - -Improved Documentation ----------------------- - -- Added bullet list of Python Plugin features and a tech preview page for new experimental features - `#7628 `_ - - ----- - - -3.0.0b12 (2020-11-05) -===================== - - -Features --------- - -- Pulp Python can now fully mirror all packages from PyPi - `#985 `_ -- Implemented PyPi's json API at content endpoint '/pypi/{package-name}/json'. Pulp can now perform basic syncing on other Pulp Python instances. - `#2886 `_ -- Pulp Python now uses Bandersnatch to perform syncing and filtering of package metadata - `#6930 `_ - - -Bugfixes --------- - -- Sync now includes python package's classifiers in the content unit - `#3627 `_ -- Policy can now be specified when creating a remote from a Bandersnatch config - `#7331 `_ -- Includes/excludes/prereleases fields are now properly set in a remote from Bandersnatch config - `#7392 `_ - - -Improved Documentation ----------------------- - -- Fixed makemigrations commands in the install docs - `#5386 `_ - - -Misc ----- - -- `#6875 `_, `#7401 `_ - - ----- - - -3.0.0b11 (2020-08-18) -===================== - - -Compatibility update for pulpcore 3.6 - - ----- - - -3.0.0b10 (2020-08-05) -===================== - - -Features --------- - -- Added a new endpoint to remotes "/from_bandersnatch" that allows for Python remote creation from a Bandersnatch config file. - `#6929 `_ - - -Bugfixes --------- - -- Including requirements.txt on MANIFEST.in - `#6891 `_ -- Updating API to not return publications that aren't complete. - `#6987 `_ -- Fixed an issue that prevented 'on_demand' content from being published. - `#7128 `_ - - -Improved Documentation ----------------------- - -- Change the commands for publication and distribution on the publish workflow to use their respective scripts already defined in _scripts. - `#6877 `_ -- Updated sync.sh, publication.sh and distribution.sh in docs/_scripts to reference wait_until_task_finished function from base.sh - `#6918 `_ - - ----- - - -3.0.0b9 (2020-06-01) -==================== - - -Features --------- - -- Add upload functionality to the python contents endpoints. - `#5464 `_ - - -Bugfixes --------- - -- Fixed the 500 error returned by the OpenAPI schema endpoint. - `#5452 `_ - - -Improved Documentation ----------------------- - -- Change the prefix of Pulp services from pulp-* to pulpcore-* - `#4554 `_ -- Added "python/python/" to fix two commands in repo.sh, fixed export command in sync.sh - `#6790 `_ -- Added "index.html" to the relative_path field for both project_metadata and index_metadata. Added a "/" to fix the link in the simple_index_template. - `#6792 `_ -- Updated the workflow documentation for upload.html. Fixed the workflow commands and added more details to the instructions. - `#6854 `_ - - -Deprecations and Removals -------------------------- - -- Change `_id`, `_created`, `_last_updated`, `_href` to `pulp_id`, `pulp_created`, `pulp_last_updated`, `pulp_href` - `#5457 `_ -- Remove "_" from `_versions_href`, `_latest_version_href` - `#5548 `_ -- Removing base field: `_type` . - `#5550 `_ -- Sync is no longer available at the {remote_href}/sync/ repository={repo_href} endpoint. Instead, use POST {repo_href}/sync/ remote={remote_href}. - - Creating / listing / editing / deleting python repositories is now performed on /pulp/api/v3/python/python/ instead of /pulp/api/v3/repositories/. Only python content can be present in a python repository, and only a python repository can hold python content. - `#5625 `_ - - -Misc ----- - -- `#remotetests `_, `#4681 `_, `#4682 `_, `#5304 `_, `#5471 `_, `#5580 `_, `#5701 `_ - - ----- - - -3.0.0b8 (2019-09-16) -==================== - - -Misc ----- - -- `#4681 `_ - - ----- - - -3.0.0b7 (2019-08-01) -==================== - - -Features --------- - -- Users can upload a file to create content and optionally add to a repo in one step known as - one-shot upload - `#4396 `_ -- Override the Remote's serializer to allow policy='on_demand' and policy='streamed'. - `#4990 `_ - - -Improved Documentation ----------------------- - -- Switch to using `towncrier `_ for better release notes. - `#4875 `_ - - ----- - - diff --git a/CHANGES/+pulpcore-3.40.feature b/CHANGES/+pulpcore-3.40.feature deleted file mode 100644 index 7aa25a62e..000000000 --- a/CHANGES/+pulpcore-3.40.feature +++ /dev/null @@ -1 +0,0 @@ -Added pulpcore 3.40 compatibility. diff --git a/CHANGES/.TEMPLATE.rst b/CHANGES/.TEMPLATE.md similarity index 64% rename from CHANGES/.TEMPLATE.rst rename to CHANGES/.TEMPLATE.md index ab3826e76..2308193b9 100644 --- a/CHANGES/.TEMPLATE.rst +++ b/CHANGES/.TEMPLATE.md @@ -1,37 +1,39 @@ - {# TOWNCRIER TEMPLATE #} {% for section, _ in sections.items() %} -{% set underline = underlines[0] %}{% if section %}{{section}} -{{ underline * section|length }}{% set underline = underlines[1] %} +{%- set section_slug = "-" + section|replace(" ", "-")|replace("_", "-")|lower %} +{%- if section %} +### {{section}} {: #{{versiondata.version}}{{section_slug}} } +{% else %} +{%- set section_slug = "" %} {% endif %} - {% if sections[section] %} {% for category, val in definitions.items() if category in sections[section]%} -{{ definitions[category]['name'] }} -{{ underline * definitions[category]['name']|length }} + +#### {{ definitions[category]['name'] }} {: #{{versiondata.version}}{{section_slug}}-{{category}} } {% if definitions[category]['showcontent'] %} {% for text, values in sections[section][category].items() %} - {{ text }} +{% if values %} {{ values|join(',\n ') }} +{% endif %} {% endfor %} - {% else %} - {{ sections[section][category]['']|join(', ') }} - {% endif %} {% if sections[section][category]|length == 0 %} -No significant changes. +No significant changes. {% else %} {% endif %} - {% endfor %} {% else %} -No significant changes. - +No significant changes. {% endif %} {% endfor %} ----- + +--- + + diff --git a/CHANGES/579.feature b/CHANGES/579.feature deleted file mode 100644 index 19390c766..000000000 --- a/CHANGES/579.feature +++ /dev/null @@ -1 +0,0 @@ -Added import export support of python content. diff --git a/MANIFEST.in b/MANIFEST.in index aca0d5946..e519d8a0a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,11 @@ include LICENSE include requirements.txt include pyproject.toml -include CHANGES.rst +include CHANGES.md include COMMITMENT include COPYRIGHT include functest_requirements.txt include test_requirements.txt include unittest_requirements.txt include pulp_python/app/webserver_snippets/* +exclude releasing.md diff --git a/doc_requirements.txt b/doc_requirements.txt index 925c79a06..6456b7d8d 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,14 +1,8 @@ -build -coreapi -django -djangorestframework -django-filter -drf-nested-routers -mistune -plantuml -pyyaml -sphinx -sphinx-rtd-theme -sphinxcontrib-openapi +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template towncrier -twine +pulp-docs @ git+https://github.com/pulp/pulp-docs@main diff --git a/docs/_scripts/base.sh b/docs/_scripts/base.sh index b80b5d101..a1b45744a 100755 --- a/docs/_scripts/base.sh +++ b/docs/_scripts/base.sh @@ -25,5 +25,7 @@ if [ ! -f ~/.config/pulp/settings.toml ]; then base_url = "$BASE_ADDR" verify_ssl = false format = "json" +username = "admin" +password = "password" EOF fi diff --git a/docs/changes.rst b/docs/changes.rst index 3e96678c2..96e8eb90c 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,5 +1,4 @@ -.. _pulp_python-changes: +Changes +********* -.. include:: ../CHANGES.rst - -.. include:: ../HISTORY.rst +Removed due to docs migration process. diff --git a/docs/conf.py b/docs/conf.py index 17fcf22ee..3971ab459 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.11.0.dev" +version = "3.11.4.dev" # The full version, including alpha/beta/rc tags. -release = "3.11.0.dev" +release = "3.11.4.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/functest_requirements.txt b/functest_requirements.txt index c43d8b3d7..64118294b 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -1,7 +1,7 @@ git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash -pytest +pytest<8 lxml twine pypi-simple -pulpcore-client -pulp-python-client +pytest-xdist +pytest-timeout diff --git a/lint_requirements.txt b/lint_requirements.txt index 4e7958f12..a29362097 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,7 +5,8 @@ # # For more info visit https://github.com/pulp/plugin_template -# python packages handy for developers, but not required by pulp +bump-my-version check-manifest flake8 - +packaging +yamllint diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index b5227fa83..d008fbc29 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.0.dev" + version = "3.11.8.dev" python_package_name = "pulp-python" diff --git a/pulp_python/app/fields.py b/pulp_python/app/fields.py new file mode 100644 index 000000000..110d2c55c --- /dev/null +++ b/pulp_python/app/fields.py @@ -0,0 +1,12 @@ +from drf_spectacular.utils import extend_schema_field +from drf_spectacular.types import OpenApiTypes +from rest_framework import serializers + + +@extend_schema_field(OpenApiTypes.OBJECT) +class JSONObjectField(serializers.JSONField): + """A drf JSONField override to force openapi schema to use 'object' type. + + Not strictly correct, but we relied on that for a long time. + See: https://github.com/tfranzel/drf-spectacular/issues/1095 + """ diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 0e8b29fc7..1adcfc74c 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -4,7 +4,6 @@ from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ObjectDoesNotExist from django.db import models -from django.conf import settings from pulpcore.plugin.models import ( Content, Publication, @@ -13,9 +12,11 @@ Repository, ) from pulpcore.plugin.responses import ArtifactResponse +from pulpcore.plugin.util import get_domain from pathlib import PurePath from .utils import ( + canonicalize_name, get_project_metadata_from_artifact, parse_project_metadata, python_content_to_json, @@ -76,7 +77,7 @@ def content_handler(self, path): elif len(path.parts) and path.parts[0] == "simple": # Temporary fix for PublishedMetadata not being properly served from remote storage # https://github.com/pulp/pulp_python/issues/413 - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if get_domain().storage_class != "pulpcore.app.models.storage.FileSystem": if self.publication or self.repository: try: publication = self.publication or Publication.objects.filter( @@ -84,6 +85,8 @@ def content_handler(self, path): ).latest("pulp_created") except ObjectDoesNotExist: return None + if len(path.parts) == 2: + path = PurePath(f"simple/{canonicalize_name(path.parts[1])}") rel_path = f"{path}/index.html" try: ca = ( @@ -100,8 +103,9 @@ def content_handler(self, path): return ArtifactResponse(ca.artifact, headers=headers) if name: + normalized = canonicalize_name(name) package_content = PythonPackageContent.objects.filter( - pk__in=self.publication.repository_version.content, name__iexact=name + pk__in=self.publication.repository_version.content, name__normalize=normalized ) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index d5f6d6073..906293afb 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -3,6 +3,7 @@ from rest_framework import serializers from pulp_python.app.utils import DIST_EXTENSIONS +from pulp_python.app import fields from pulpcore.plugin.models import Artifact from django.db.utils import IntegrityError @@ -27,9 +28,9 @@ class PackageMetadataSerializer(serializers.Serializer): """ last_serial = serializers.IntegerField(help_text=_("Cache value from last PyPI sync")) - info = serializers.JSONField(help_text=_("Core metadata of the package")) - releases = serializers.JSONField(help_text=_("List of all the releases of the package")) - urls = serializers.JSONField() + info = fields.JSONObjectField(help_text=_("Core metadata of the package")) + releases = fields.JSONObjectField(help_text=_("List of all the releases of the package")) + urls = fields.JSONObjectField() class PackageUploadSerializer(serializers.Serializer): diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index ab2765580..d2e34f6bf 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -278,7 +278,8 @@ def retrieve(self, request, path, meta): elif meta_path.match("*/json"): name = meta_path.parts[0] if name: - package_content = content.filter(name__iexact=name) + normalized = canonicalize_name(name) + package_content = content.filter(name__normalize=normalized) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} json_body = python_content_to_json(path, package_content, version=version) diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 2573a1e1b..36fb6395e 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -7,6 +7,7 @@ from pulpcore.plugin import serializers as core_serializers from pulp_python.app import models as python_models +from pulp_python.app import fields from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata @@ -154,7 +155,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_('A browsable URL for the project and a label for it, separated by a comma.') ) - project_urls = serializers.JSONField( + project_urls = fields.JSONObjectField( required=False, default=dict, help_text=_('A dictionary of labels and URLs for the project.') ) @@ -167,28 +168,28 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ') ) - requires_dist = serializers.JSONField( + requires_dist = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing names of some other distutils project ' 'required by this distribution.') ) - provides_dist = serializers.JSONField( + provides_dist = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing names of a Distutils project which is contained' ' within this distribution.') ) - obsoletes_dist = serializers.JSONField( + obsoletes_dist = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing names of a distutils project\'s distribution which ' 'this distribution renders obsolete, meaning that the two projects should not ' 'be installed at the same time.') ) - requires_external = serializers.JSONField( + requires_external = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing some dependency in the system that the distribution ' 'is to be used.') ) - classifiers = serializers.JSONField( + classifiers = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing classification values for a Python package.') ) diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index c4f93dfa9..1293c6130 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -1,9 +1,12 @@ import logging +import tempfile +from typing import Optional, Any, AsyncGenerator +import aiohttp from aiohttp import ClientResponseError, ClientError from lxml.etree import LxmlError from gettext import gettext as _ -from os import environ, path +from os import environ from rest_framework import serializers @@ -80,7 +83,7 @@ def create_bandersnatch_config(remote): config["plugins"]["enabled"] += "prerelease_release\n" if remote.package_types: rrfm = "regex_release_file_metadata" - config["plugins"]["enabled"] += rrfm + config["plugins"]["enabled"] += f"{rrfm}\n" if not config.has_section(rrfm): config.add_section(rrfm) config[rrfm]["any:release_file.packagetype"] = "\n".join(remote.package_types) @@ -111,9 +114,10 @@ async def run(self): """ If includes is specified, then only sync those,else try to sync all other packages """ - # Prevent bandersnatch from reading actual .netrc file, set to nonexistent file + # Prevent bandersnatch from reading actual .netrc file, set to empty file # See discussion on https://github.com/pulp/pulp_python/issues/581 - environ["NETRC"] = f"{path.curdir}/.fake-netrc" + fake_netrc = tempfile.NamedTemporaryFile(dir=".", delete=False) + environ["NETRC"] = fake_netrc.name # TODO Change Bandersnatch internal API to take proxy settings in from config parameters if proxy_url := self.remote.proxy_url: if self.remote.proxy_username or self.remote.proxy_password: @@ -126,7 +130,7 @@ async def run(self): # Bandersnatch includes leading slash when forming API urls url = self.remote.url.rstrip("/") # local & global timeouts defaults to 10secs and 5 hours - async with Master(url) as master: + async with PulpMaster(url, tls=self.remote.tls_validation) as master: deferred_download = self.remote.policy != Remote.IMMEDIATE workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY async with ProgressReport( @@ -148,6 +152,25 @@ async def run(self): await pmirror.synchronize(packages_to_sync) +class PulpMaster(Master): + """ + Pulp Master Class for Pulp specific overrides + """ + + def __init__(self, *args, tls=True, **kwargs): + self.tls = tls + super().__init__(*args, **kwargs) + + async def get( + self, path: str, required_serial: Optional[int], **kw: Any + ) -> AsyncGenerator[aiohttp.ClientResponse, None]: + """Support tls=false""" + if not self.tls: + kw["ssl"] = False + async for r in super().get(path, required_serial, **kw): + yield r + + class PulpMirror(Mirror): """ Pulp Mirror Class to perform syncing using Bandersnatch @@ -257,4 +280,4 @@ def on_error(self, exception, **kwargs): TODO This should have some error checking """ - pass + logger.error("Sync encountered an error: ", exc_info=exception) diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index eb1082de3..db6acdd09 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -3,7 +3,6 @@ import tempfile import json from collections import defaultdict -from django.core.files.storage import default_storage as storage from django.conf import settings from jinja2 import Template from packaging.utils import canonicalize_name @@ -144,8 +143,8 @@ def get_project_metadata_from_artifact(filename, artifact): # because pkginfo validates that the filename has a valid extension before # reading it with tempfile.NamedTemporaryFile('wb', dir=".", suffix=filename) as temp_file: - artifact_file = storage.open(artifact.file.name) - shutil.copyfileobj(artifact_file, temp_file) + artifact.file.seek(0) + shutil.copyfileobj(artifact.file, temp_file) temp_file.flush() metadata = DIST_TYPES[packagetype](temp_file.name) metadata.packagetype = packagetype diff --git a/pulp_python/tests/functional/api/test_consume_content.py b/pulp_python/tests/functional/api/test_consume_content.py index dca407280..a46358be3 100644 --- a/pulp_python/tests/functional/api/test_consume_content.py +++ b/pulp_python/tests/functional/api/test_consume_content.py @@ -122,6 +122,7 @@ def install(cli_client, package, host=PYPI_URL): "pip", "install", "--no-deps", + "--no-build-isolation", "--trusted-host", urlsplit(host).hostname, "--trusted-host", diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index 2f385f3cd..f3dbe073b 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -1,5 +1,6 @@ # coding=utf-8 """Tests that perform actions over content unit.""" +import pytest from pulp_smash.pulp3.bindings import delete_orphans, monitor_task, PulpTaskError from pulp_python.tests.functional.utils import ( @@ -12,6 +13,7 @@ from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 from tempfile import NamedTemporaryFile from urllib.parse import urljoin +from pypi_simple import PyPISimple from pulp_smash.utils import http_get from pulp_python.tests.functional.constants import ( @@ -227,3 +229,29 @@ def check_package_data(self, content_unit, expected=PYTHON_PACKAGE_DATA): for k, v in expected.items(): with self.subTest(key=k): self.assertEqual(content_unit[k], v) + + +@pytest.mark.parallel +def test_upload_metadata_23_spec(python_content_factory): + """Test that packages using metadata spec 2.3 can be uploaded to pulp.""" + filename = "urllib3-2.2.2-py3-none-any.whl" + with PyPISimple() as client: + page = client.get_project_page("urllib3") + for package in page.packages: + if package.filename == filename: + content = python_content_factory(filename, url=package.url) + assert content.metadata_version == "2.3" + break + + +@pytest.mark.parallel +def test_upload_metadata_24_spec(python_content_factory): + """Test that packages using metadata spec 2.4 can be uploaded to pulp.""" + filename = "urllib3-2.3.0-py3-none-any.whl" + with PyPISimple() as client: + page = client.get_project_page("urllib3") + for package in page.packages: + if package.filename == filename: + content = python_content_factory(filename, url=package.url) + assert content.metadata_version == "2.4" + break diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 2f20cca0a..236f21070 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -1,5 +1,6 @@ # coding=utf-8 """Tests that verify download of content served by Pulp.""" +import pytest import hashlib from random import choice from urllib.parse import urljoin @@ -149,3 +150,32 @@ def test_full_pulp_to_pulp_sync(self): repo3 = self._create_repo_and_sync_with_remote(remote) self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_MD_FIXTURE_SUMMARY) + + +@pytest.mark.parallel +def test_pulp2pulp_sync_with_oddities( + python_repo_with_sync, + python_remote_factory, + python_publication_factory, + python_distribution_factory, + python_content_summary, +): + """Test that Pulp can handle syncing packages with wierd names.""" + remote = python_remote_factory(includes=["oslo.utils"], url="iframe.php?url=https%3A%2F%2Fpypi.org") + repo = python_repo_with_sync(remote) + distro = python_distribution_factory(repository=repo.pulp_href) + summary = python_content_summary(repository_version=repo.latest_version_href) + # Test pulp 2 pulp full sync w/ live pypi apis + remote2 = python_remote_factory(includes=[], url=distro.base_url) + repo2 = python_repo_with_sync(remote2) + summary2 = python_content_summary(repository_version=repo2.latest_version_href) + assert summary2.present["python.python"]["count"] > 0 + assert summary.present["python.python"]["count"] == summary2.present["python.python"]["count"] + # Test w/ publication + pub = python_publication_factory(repository=repo) + distro2 = python_distribution_factory(publication=pub.pulp_href) + remote3 = python_remote_factory(includes=[], url=distro2.base_url) + repo3 = python_repo_with_sync(remote3) + summary3 = python_content_summary(repository_version=repo3.latest_version_href) + assert summary3.present["python.python"]["count"] > 0 + assert summary.present["python.python"]["count"] == summary3.present["python.python"]["count"] diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 2cab07f21..7e6fa6261 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -3,10 +3,11 @@ import requests import subprocess import tempfile +import pytest from urllib.parse import urljoin -from pulp_smash.pulp3.bindings import monitor_task, tasks as task_api +from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import get_added_content_summary, get_content_summary from pulp_python.tests.functional.constants import ( PYTHON_CONTENT_NAME, @@ -40,6 +41,32 @@ PYPI_HOST = urljoin(HOST, PULP_PYPI_BASE_URL) +@pytest.fixture +def python_empty_repo_distro(python_repo_factory, python_distribution_factory): + """Returns an empty repo with and distribution serving it.""" + def _generate_empty_repo_distro(repo_body=None, distro_body=None): + repo_body = repo_body or {} + distro_body = distro_body or {} + repo = python_repo_factory(**repo_body) + distro = python_distribution_factory(repository=repo.pulp_href, **distro_body) + return repo, distro + + yield _generate_empty_repo_distro + + +@pytest.fixture(scope="module") +def python_package_dist_directory(tmp_path_factory, http_get): + """Creates a temp dir to hold package distros for uploading.""" + dist_dir = tmp_path_factory.mktemp("dist") + egg_file = dist_dir / PYTHON_EGG_FILENAME + wheel_file = dist_dir / PYTHON_WHEEL_FILENAME + with open(egg_file, "wb") as f: + f.write(http_get(PYTHON_EGG_URL)) + with open(wheel_file, "wb") as f: + f.write(http_get(PYTHON_WHEEL_URL)) + yield dist_dir, egg_file, wheel_file + + class PyPISummaryTestCase(TestCaseUsingBindings, TestHelpersMixin): """Tests the summary response of the base url of an index.""" @@ -162,70 +189,59 @@ def test_package_upload_simple(self): content = get_added_content_summary(repo, f"{repo.versions_href}1/") self.assertDictEqual({PYTHON_CONTENT_NAME: 1}, content) - def test_twine_upload(self): - """Tests that packages can be properly uploaded through Twine.""" - repo, distro = self._create_empty_repo_and_distribution() - url = urljoin(PYPI_HOST, distro.base_path + "/legacy/") - username, password = "admin", "password" - subprocess.run( - ( - "twine", - "upload", - "--repository-url", - url, - self.dists_dir.name + "/*", - "-u", - username, - "-p", - password, - ), - capture_output=True, - check=True, - ) - tasks = task_api.list(reserved_resources_record=[repo.pulp_href]).results - for task in reversed(tasks): - t = monitor_task(task.pulp_href) - repo_ver_href = t.created_resources[-1] - content = get_content_summary(repo, f"{repo_ver_href}") - self.assertDictEqual({PYTHON_CONTENT_NAME: 2}, content) - - # Test re-uploading same packages gives error - with self.assertRaises(subprocess.CalledProcessError): - subprocess.run( - ( - "twine", - "upload", - "--repository-url", - url, - self.dists_dir.name + "/*", - "-u", - username, - "-p", - password, - ), - capture_output=True, - check=True, - ) - # Test re-uploading same packages with --skip-existing works - output = subprocess.run( +@pytest.mark.parallel +def test_twine_upload( + tasks_api_client, + python_content_summary, + python_empty_repo_distro, + python_package_dist_directory, + monitor_task, +): + """Tests that packages can be properly uploaded through Twine.""" + repo, distro = python_empty_repo_distro() + url = urljoin(distro.base_url, "legacy/") + dist_dir, _, _ = python_package_dist_directory + username, password = "admin", "password" + subprocess.run( + ( + "twine", + "upload", + "--repository-url", + url, + dist_dir / "*", + "-u", + username, + "-p", + password, + ), + capture_output=True, + check=True, + ) + tasks = tasks_api_client.list(reserved_resources=repo.pulp_href).results + for task in reversed(tasks): + t = monitor_task(task.pulp_href) + repo_ver_href = t.created_resources[-1] + summary = python_content_summary(repository_version=repo_ver_href) + assert summary.present["python.python"]["count"] == 2 + + # Test re-uploading same packages gives error + with pytest.raises(subprocess.CalledProcessError): + subprocess.run( ( "twine", "upload", "--repository-url", url, - self.dists_dir.name + "/*", + dist_dir / "*", "-u", username, "-p", password, - "--skip-existing", ), capture_output=True, check=True, - text=True ) - self.assertEqual(output.stdout.count("Skipping"), 2) class PyPISimpleApi(TestCaseUsingBindings, TestHelpersMixin): diff --git a/pulp_python/tests/functional/api/test_sync.py b/pulp_python/tests/functional/api/test_sync.py index b64e84041..79a07b0c9 100644 --- a/pulp_python/tests/functional/api/test_sync.py +++ b/pulp_python/tests/functional/api/test_sync.py @@ -641,6 +641,23 @@ def test_no_platform_sync(self): ) +@pytest.mark.parallel +def test_sync_multiple_filters( + python_repo_with_sync, python_remote_factory, python_content_summary +): + """Tests sync with multiple filters.""" + remote = python_remote_factory( + includes=PYTHON_LG_PROJECT_SPECIFIER, + package_types=["bdist_wheel"], + keep_latest_packages=1, + prereleases=False + ) + repo = python_repo_with_sync(remote) + + summary = python_content_summary(repository_version=repo.latest_version_href) + assert summary.present["python.python"]["count"] == PYTHON_LG_FIXTURE_COUNTS["multi"] + + @pytest.mark.parallel def test_proxy_sync( python_repo, diff --git a/pulp_python/tests/functional/conftest.py b/pulp_python/tests/functional/conftest.py index 01a0552da..732695e44 100644 --- a/pulp_python/tests/functional/conftest.py +++ b/pulp_python/tests/functional/conftest.py @@ -1,8 +1,10 @@ import pytest +import subprocess import uuid from pulp_smash.pulp3.utils import gen_distribution from pulp_python.tests.functional.utils import gen_python_remote +from pulp_python.tests.functional.constants import PYTHON_URL, PYTHON_EGG_FILENAME from pulpcore.client.pulp_python import ( ApiClient, @@ -110,3 +112,93 @@ def _gen_python_remote(**kwargs): return gen_object_with_cleanup(python_remote_api_client, body) yield _gen_python_remote + + +@pytest.fixture +def python_repo_with_sync( + python_repo_api_client, python_repo_factory, python_remote_factory, monitor_task +): + """A factory to generate a Python Repository synced with the passed in Remote.""" + def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): + kwargs = {} + if pulp_domain := body.get("pulp_domain"): + kwargs["pulp_domain"] = pulp_domain + remote = remote or python_remote_factory(**kwargs) + repo = repository or python_repo_factory(**body) + sync_body = {"mirror": mirror, "remote": remote.pulp_href} + monitor_task(python_repo_api_client.sync(repo.pulp_href, sync_body).task) + return python_repo_api_client.read(repo.pulp_href) + + yield _gen_python_repo_sync + + +@pytest.fixture +def download_python_file(tmp_path, http_get): + """Download a Python file and return its path.""" + def _download_python_file(relative_path, url): + file_path = tmp_path / relative_path + with open(file_path, mode="wb") as f: + f.write(http_get(url)) + return file_path + + yield _download_python_file + + +@pytest.fixture +def python_file(download_python_file): + """Get a default (shelf-reader.tar.gz) Python file.""" + return download_python_file(PYTHON_EGG_FILENAME, PYTHON_URL) + + +@pytest.fixture +def python_content_factory(python_content_api_client, download_python_file, monitor_task): + """A factory to create a Python Package Content.""" + def _gen_python_content(relative_path=PYTHON_EGG_FILENAME, url=None, **body): + body["relative_path"] = relative_path + if url: + body["file"] = download_python_file(relative_path, url) + elif not any(x in body for x in ("artifact", "file", "upload")): + body["file"] = download_python_file(PYTHON_EGG_FILENAME, PYTHON_URL) + if repo := body.get("repository"): + repo_href = repo if isinstance(repo, str) else repo.pulp_href + body["repository"] = repo_href + + task = python_content_api_client.create(**body).task + response = monitor_task(task) + return python_content_api_client.read(response.created_resources[0]) + + yield _gen_python_content + + +# Utility fixtures + + +@pytest.fixture +def shelf_reader_cleanup(): + """Take care of uninstalling shelf-reader before/after the test.""" + cmd = ("pip", "uninstall", "shelf-reader", "-y") + subprocess.run(cmd) + yield + subprocess.run(cmd) + + +@pytest.fixture +def python_content_summary(python_repo_api_client, python_repo_version_api_client): + """Get a summary of the repository version's content.""" + def _gen_summary(repository_version=None, repository=None, version=None): + if repository_version is None: + repo_href = get_href(repository) + if version: + repo_ver_href = f"{repo_href}versions/{version}/" + else: + repo_ver_href = python_repo_api_client.read(repo_href).latest_version_href + else: + repo_ver_href = get_href(repository_version) + return python_repo_version_api_client.read(repo_ver_href).content_summary + + yield _gen_summary + + +def get_href(item): + """Tries to get the href from the given item, whether it is a string or object.""" + return item if isinstance(item, str) else item.pulp_href diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index d37c8f070..f0e53046f 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -144,17 +144,22 @@ "aiohttp", # matches 7 "bcrypt", # matches 8 "celery", # matches 13 + "crane", # matches 0 "Django", # matches 31 + "pulp-2to3-migration", # matches 2 "pytz", # matches 6 "scipy", # matches 23 + "setuptools", # matches 2 "shelf-reader", # matches 2 + "twine", # matches 14 ] -PYTHON_LG_PACKAGE_COUNT = 90 +PYTHON_LG_PACKAGE_COUNT = 108 PYTHON_LG_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_LG_PACKAGE_COUNT} PYTHON_LG_FIXTURE_COUNTS = { - "latest_3": 49, - "sdist": 27, - "bdist_wheel": 63, + "latest_3": 59, + "sdist": 36, + "bdist_wheel": 72, + "multi": 36, # keep_latest=1, package_types="bdist_wheel", prereleases=False } DJANGO_LATEST_3 = 4 # latest version has 2 dists, each other has 1 diff --git a/pyproject.toml b/pyproject.toml index cba5c3f37..d30d6c218 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,12 @@ [tool.towncrier] package = "pulp_python" -filename = "CHANGES.rst" +filename = "CHANGES.md" directory = "CHANGES/" -title_format = "{version} ({project_date})" -template = "CHANGES/.TEMPLATE.rst" -issue_format = "`#{issue} `__" +title_format = "## {version} ({project_date}) {{: #{version} }}" +template = "CHANGES/.TEMPLATE.md" +issue_format = "[#{issue}](https://github.com/pulp/pulp_python/issues/{issue})" +start_string = "[//]: # (towncrier release notes start)\n" +underlines = ["", "", ""] [tool.check-manifest] ignore = [ @@ -24,4 +26,51 @@ ignore = [ ".ci/**", "lint_requirements.txt", ".flake8", + "AGENTS.md", + "CLAUDE.md", ] + +[tool.bumpversion] +# This section is managed by the plugin template. Do not edit manually. + +current_version = "3.11.8.dev" +commit = false +tag = false +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" +serialize = [ + "{major}.{minor}.{patch}.{release}", + "{major}.{minor}.{patch}", + "{major}.{minor}.{alpha}{patch}.{release}", + "{major}.{minor}.{alpha}{patch}", +] + +[tool.bumpversion.parts.alpha] +# This section is managed by the plugin template. Do not edit manually. + +# This is sort of a hack. In PEP440 prerelease markers work quite differently. +# But this fits best with the way we have been doing release versions. +optional_value = "final" +values = [ + "0a", + "final", +] +independent = true + +[tool.bumpversion.parts.release] +# This section is managed by the plugin template. Do not edit manually. + +optional_value = "prod" +values = [ + "dev", + "prod", +] + +[[tool.bumpversion.files]] +# This section is managed by the plugin template. Do not edit manually. + +filename = "./pulp_python/app/__init__.py" +search = "version = \"{current_version}\"" +replace = "version = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "./setup.py" diff --git a/releasing.md b/releasing.md new file mode 100644 index 000000000..db7470b28 --- /dev/null +++ b/releasing.md @@ -0,0 +1,29 @@ +[//]: # "WARNING: DO NOT EDIT!" +[//]: # "" +[//]: # "This file was generated by plugin_template, and is managed by it. Please use" +[//]: # "'./plugin-template --github pulp_python' to update this file." +[//]: # "" +[//]: # "For more info visit https://github.com/pulp/plugin_template" +# Releasing (For Internal Use) + +This document outlines the steps to perform a release. + +### Determine if a Release is Required +- Make sure to have GitPython python package installed +- Run the release checker script: + ``` + python3 .ci/scripts/check_release.py + ``` + +### Release a New Y-version (e.g., 3.23.0) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_python/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. +- Look for the "Bump minor version" pull request and merge it. +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_python/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. + +### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_python/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. + +## Final Steps +- Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulp_python/actions/workflows/publish.yml) workflow succeeds). +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). +- [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). diff --git a/requirements.txt b/requirements.txt index bb40f6ad8..6de2c9144 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pulpcore>=3.25.0,<3.55 -pkginfo>=1.8.2,<1.9.7 +pulpcore>=3.28.0,<3.55 +pkginfo>=1.12.0,<1.13.0 bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 diff --git a/setup.py b/setup.py index 078afa7e3..ed82a6e7d 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.0.dev", + version="3.11.8.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", diff --git a/template_config.yml b/template_config.yml index 308a2a42b..4f12934a9 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,77 +1,92 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-248-g4bfc3e1 +# generated with plugin_template +# +# After editing this file please always reapply the plugin template before committing any changes. -additional_repos: [] -api_root: /pulp/ +--- +api_root: "/pulp/" black: false check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true +ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" ci_env: {} -ci_trigger: '{pull_request: {branches: [''*'']}}' -ci_update_branches: [] -ci_update_docs: false -ci_update_release_behavior: null -cli_package: pulp-cli -cli_repo: https://github.com/pulp/pulp-cli.git +ci_trigger: "{pull_request: {branches: ['*']}}" +cli_package: "pulp-cli" +cli_repo: "https://github.com/pulp/pulp-cli.git" core_import_allowed: [] deploy_client_to_pypi: true deploy_client_to_rubygems: true deploy_to_pypi: true disabled_redis_runners: [] -doc_requirements_from_pulpcore: true docker_fixtures: false -docs_test: true -extra_docs_requirements: [] +extra_files: [] flake8: true flake8_ignore: [] -github_org: pulp -issue_tracker: github -kanban: true +github_org: "pulp" +latest_release_branch: null lint_requirements: true -noissue_marker: '[noissue]' +os_required_packages: [] parallel_test_workers: 8 -plugin_app_label: python -plugin_camel: PulpPython -plugin_camel_short: Python -plugin_caps: PULP_PYTHON -plugin_caps_short: PYTHON -plugin_dash: pulp-python -plugin_dash_short: python -plugin_default_branch: main -plugin_name: pulp_python -plugin_snake: pulp_python -post_job_template: null -pre_job_template: null -publish_docs_to_pulpprojectdotorg: true +plugin_app_label: "python" +plugin_default_branch: "main" +plugin_name: "pulp_python" +plugins: + - app_label: "python" + name: "pulp_python" pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} pulp_env_s3: {} -pulp_scheme: https +pulp_scheme: "https" pulp_settings: + allowed_export_paths: "/tmp" + allowed_import_paths: "/tmp" orphan_protection_time: 0 - pypi_api_hostname: https://pulp:443 - allowed_export_paths: /tmp - allowed_import_paths: /tmp -pulp_settings_azure: null + pypi_api_hostname: "https://pulp:443" +pulp_settings_azure: + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.azure_storage.AzureStorage" + OPTIONS: + account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + account_name: "devstoreaccount1" + azure_container: "pulp-test" + connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;" + expiration_secs: 120 + location: "pulp3" + overwrite_files: true + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" pulp_settings_gcp: null -pulp_settings_s3: null -pulpprojectdotorg_key_id: null +pulp_settings_s3: + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.s3boto3.S3Boto3Storage" + OPTIONS: + access_key: "AKIAIT2Z5TDYPX3ARJBA" + addressing_style: "path" + bucket_name: "pulp3" + default_acl: "@none" + endpoint_url: "http://minio:9000" + region_name: "eu-central-1" + secret_key: "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS" + signature_version: "s3v4" + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" pydocstyle: true -pypi_username: pulp -python_version: '3.8' -release_email: pulp-infra@redhat.com -release_user: pulpbot -run_pulpcore_tests_for_plugins: false -single_commit_check: true +release_email: "pulp-infra@redhat.com" +release_user: "pulpbot" stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 stalebot_limit_to_pulls: true +supported_release_branches: [] sync_ci: true test_azure: true test_cli: true @@ -79,8 +94,7 @@ test_deprecations: true test_gcp: false test_lowerbounds: true test_performance: false -test_reroute: true test_s3: true -update_github: true use_issue_template: true +... diff --git a/unittest_requirements.txt b/unittest_requirements.txt index 93253de97..f5b662af5 100644 --- a/unittest_requirements.txt +++ b/unittest_requirements.txt @@ -1,2 +1,2 @@ mock -pytest +pytest<8