diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 01469b64..fb9ed4a1 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,34 +1,28 @@ -FROM {{ ci_base | default(pulp_default_container) }} +FROM {{ image.ci_base }} +{%- if image.webserver_snippet %} -# Add source directories to container -{% for item in plugins %} -ADD ./{{ item.name }} ./{{ item.name }} -{% endfor %} +ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf +{%- endif %} + +{%- for item in extra_files | default([]) %} -{% for item in extra_files | default([]) %} ADD ./{{ item.origin }} {{ item.destination }} -{% endfor %} +{%- endfor %} # This MUST be the ONLY call to pip install in inside the container. RUN pip3 install --upgrade pip setuptools wheel && \ rm -rf /root/.cache/pip && \ - pip3 install -{%- if s3_test | default(false) -%} -{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue -{%- endif -%} -{%- for item in plugins -%} -{{ " " }}{{ item.source }} -{%- if item.upperbounds | default(false) -%} -{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt + pip3 install {{ image.source }} +{%- if image.upperbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt {%- endif -%} -{%- if item.lowerbounds | default(false) -%} -{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt +{%- if image.lowerbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt {%- endif -%} -{%- if item.ci_requirements | default(false) -%} -{{ " " }}-r ./{{ item.name }}/ci_requirements.txt +{%- if image.ci_requirements | default(false) -%} +{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt {%- endif -%} -{%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ +{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \ rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} @@ -46,11 +40,8 @@ ENV {{ key | upper }}={{ value }} USER pulp:pulp RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link -USER root:root -{% for item in plugins %} -RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ - ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true -{% endfor %} +RUN mkdir /var/lib/pulp/.config +USER root:root ENTRYPOINT ["/init"] diff --git a/.ci/ansible/build_container.yaml b/.ci/ansible/build_container.yaml index c380b430..0a188cba 100644 --- a/.ci/ansible/build_container.yaml +++ b/.ci/ansible/build_container.yaml @@ -1,15 +1,14 @@ # Ansible playbook to create the pulp service containers image --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Generate Containerfile from template" - template: - src: Containerfile.j2 - dest: Containerfile - + ansible.builtin.template: + src: "Containerfile.j2" + dest: "Containerfile" - name: "Build pulp image" # We build from the ../.. (parent dir of pulpcore git repo) Docker build # "context" so that repos like pulp-smash are accessible to Docker diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 47e5221e..acdc22ad 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -1,33 +1,26 @@ # Ansible playbook to start the pulp service container and its supporting services --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Create Settings Directories" - file: + ansible.builtin.file: path: "{{ item }}" - state: directory + state: "directory" mode: "0755" loop: - - settings - - ssh - - ~/.config/pulp_smash + - "settings" - name: "Generate Pulp Settings" template: - src: settings.py.j2 - dest: settings/settings.py - - - name: "Configure pulp-smash" - copy: - src: smash-config.json - dest: ~/.config/pulp_smash/settings.json + src: "settings.py.j2" + dest: "settings/settings.py" - name: "Setup docker networking" docker_network: - name: pulp_ci_bridge + name: "pulp_ci_bridge" - name: "Start Service Containers" docker_container: @@ -37,24 +30,24 @@ recreate: true privileged: true networks: - - name: pulp_ci_bridge + - name: "pulp_ci_bridge" aliases: "{{ item.name }}" volumes: "{{ item.volumes | default(omit) }}" env: "{{ item.env | default(omit) }}" command: "{{ item.command | default(omit) }}" - state: started + state: "started" loop: "{{ services | default([]) }}" - name: "Retrieve Docker Network Info" docker_network_info: - name: pulp_ci_bridge - register: pulp_ci_bridge_info + name: "pulp_ci_bridge" + register: "pulp_ci_bridge_info" - name: "Update /etc/hosts" lineinfile: - path: /etc/hosts + path: "/etc/hosts" regexp: "\\s{{ item.value.Name }}\\s*$" - line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}" + line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}" loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}" become: true @@ -63,19 +56,19 @@ aws_access_key: "{{ minio_access_key }}" aws_secret_key: "{{ minio_secret_key }}" s3_url: "http://minio:9000" - region: eu-central-1 - name: pulp3 - state: present - when: s3_test | default(false) + region: "eu-central-1" + name: "pulp3" + state: "present" + when: "s3_test | default(false)" - block: - name: "Wait for Pulp" uri: url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/" - follow_redirects: all - validate_certs: no - register: result - until: result.status == 200 + follow_redirects: "all" + validate_certs: "no" + register: "result" + until: "result.status == 200" retries: 12 delay: 5 rescue: @@ -86,7 +79,7 @@ - name: "Check version of component being tested" assert: that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver) + - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)" fail_msg: | Component {{ item.app_label }} was expected to be installed in version {{ component_version }}. Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}. @@ -100,9 +93,20 @@ login admin password password -- hosts: pulp +- hosts: "pulp" gather_facts: false tasks: + - name: "Create directory for pulp-smash config" + ansible.builtin.file: + path: "/var/lib/pulp/.config/pulp_smash/" + state: "directory" + mode: "0755" + + - name: "Configure pulp-smash" + ansible.builtin.copy: + src: "smash-config.json" + dest: "/var/lib/pulp/.config/pulp_smash/settings.json" + - name: "Set pulp admin password" command: cmd: "pulpcore-manager reset-admin-password --password password" diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 86e250e4..611c882c 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -11,7 +11,9 @@ import argparse import re import os +import sys import tomllib +import typing as t from pathlib import Path import yaml @@ -23,7 +25,7 @@ Z_CHANGELOG_EXTS = [".bugfix", ".misc"] -def options(): +def options() -> argparse.Namespace: """Check which branches need a release.""" parser = argparse.ArgumentParser() parser.add_argument( @@ -42,13 +44,13 @@ def options(): return parser.parse_args() -def template_config(): +def template_config() -> dict[str, t.Any]: # Assume this script lies in .ci/scripts path = Path(__file__).absolute().parent.parent.parent / "template_config.yml" return yaml.safe_load(path.read_text()) -def current_version(repo, commitish): +def current_version(repo: Repo, commitish: str) -> Version: try: pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml")) try: @@ -62,7 +64,7 @@ def current_version(repo, commitish): return Version(current_version) -def check_pyproject_dependencies(repo, from_commit, to_commit): +def check_pyproject_dependencies(repo: Repo, from_commit: str, to_commit: str) -> list[str]: try: new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml")) try: @@ -83,8 +85,8 @@ def check_pyproject_dependencies(repo, from_commit, to_commit): return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."] -def main(options, template_config): - DEFAULT_BRANCH = template_config["plugin_default_branch"] +def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int: + DEFAULT_BRANCH: str = template_config["plugin_default_branch"] repo = Repo() @@ -97,7 +99,7 @@ def main(options, template_config): # Warning: This will not work if branch names contain "/" but we don't really care here. heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")] - available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] + available_branches = [h for h in heads if re.fullmatch(RELEASE_BRANCH_REGEX, h)] available_branches.sort(key=lambda ver: Version(ver)) available_branches.append(DEFAULT_BRANCH) @@ -114,7 +116,10 @@ def main(options, template_config): if diff := branches - set(available_branches): print(f"Supplied branches contains non-existent branches! {diff}") - exit(1) + return 1 + + branches = [branch for branch in available_branches if branch in branches] + branches.reverse() print(f"Checking for releases on branches: {branches}") @@ -179,6 +184,8 @@ def main(options, template_config): if len(releases) == 0: print("No new releases to perform.") + return 0 + if __name__ == "__main__": - main(options(), template_config()) + sys.exit(main(options(), template_config())) diff --git a/.ci/scripts/clean_gh_release_notes.py b/.ci/scripts/clean_gh_release_notes.py new file mode 100755 index 00000000..2bf5f4c2 --- /dev/null +++ b/.ci/scripts/clean_gh_release_notes.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# This script is running with elevated privileges from the main branch against pull requests. +# +# It cleans the input from artifacts which are used by the pulp documentation internally, +# but clutter for GitHub releases + +import sys + +NOTE = """ +> [!NOTE] +> Official changes are available on [Pulp docs]({docs_url})\ +""" + + +def main(): + plugin_name = sys.argv[1] + version_str = sys.argv[2] + docs_url = f"https://pulpproject.org/{plugin_name}/changes/#{version_str}" + note_added = False + for line in sys.stdin: + if line.endswith("\n"): + line = line[:-1] + if line.startswith("#"): + print(line.split(" {: #")[0]) + if not note_added and version_str in line: + print(NOTE.format(docs_url=docs_url)) + note_added = True + else: + print(line) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 877ebc8c..fbb5d59d 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -1,4 +1,12 @@ #!/bin/env python3 +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# ] +# /// + # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 67878dc4..988902a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,23 +19,21 @@ jobs: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install build packaging twine wheel mkdocs jq - echo ::endgroup:: - name: "Build package" run: | python3 -m build @@ -48,7 +46,7 @@ jobs: pulpcore-manager openapi --file "api.json" pulpcore-manager openapi --bindings --component "python" --file "python-api.json" - name: "Upload Package whl" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "plugin_package" path: "pulp_python/dist/" @@ -56,7 +54,7 @@ jobs: retention-days: 5 overwrite: true - name: "Upload API specs" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "api_spec" path: | @@ -75,7 +73,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - name: "Upload python client packages" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "python-client.tar" path: | @@ -84,7 +82,7 @@ jobs: retention-days: 5 overwrite: true - name: "Upload python client docs" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "python-client-docs.tar" path: | @@ -102,7 +100,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - name: "Upload Ruby client" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "ruby-client.tar" path: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c4bcf91..2965afca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,18 +21,16 @@ jobs: check-commits: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install requests pygithub pyyaml - echo ::endgroup:: - name: "Check commit message" if: github.event_name == 'pull_request' env: @@ -44,29 +42,27 @@ jobs: .github/workflows/scripts/check_commit.sh check-changes: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" outputs: - run_tests: ${{ steps.check.outputs.run_tests }} - run_docs: ${{ steps.check.outputs.run_docs }} + run_tests: "${{ steps.check.outputs.run_tests }}" + run_docs: "${{ steps.check.outputs.run_docs }}" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.12" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install gitpython - echo ::endgroup:: - - name: Analyze changed files - shell: bash - id: check + - name: "Analyze changed files" + id: "check" + shell: "bash" run: | # We only test docs on the default branch (usually main) if [[ "${{ github.base_ref }}" == *"main" ]]; then @@ -91,16 +87,19 @@ jobs: needs: "check-changes" uses: "./.github/workflows/docs.yml" with: - run_docs: ${{ needs.check-changes.outputs.run_docs }} + run_docs: "${{ needs.check-changes.outputs.run_docs }}" lint: uses: "./.github/workflows/lint.yml" + sanity: + uses: "./.github/workflows/sanity.yml" + build: needs: - "check-changes" - "lint" - if: needs.check-changes.outputs.run_tests == '1' + if: "needs.check-changes.outputs.run_tests == '1'" uses: "./.github/workflows/build.yml" test: @@ -112,7 +111,7 @@ jobs: deprecations: runs-on: "ubuntu-latest" - if: github.base_ref == 'main' + if: "github.base_ref == 'main'" needs: "test" steps: - name: "Create working directory" @@ -120,7 +119,7 @@ jobs: mkdir -p "pulp_python" working-directory: "." - name: "Download Deprecations" - uses: actions/download-artifact@v4 + uses: "actions/download-artifact@v8" with: pattern: "deprecations-*" path: "pulp_python" @@ -139,6 +138,7 @@ jobs: - "lint" - "test" - "docs" + - "sanity" if: "always()" steps: - name: "Collect needed jobs results" @@ -171,3 +171,4 @@ jobs: check_jobs "$FILTERS" echo "CI says: Looks good!" +... diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 899147c4..22e5d99b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,39 +4,42 @@ # './plugin-template --github pulp_python' to update this file. # # For more info visit https://github.com/pulp/plugin_template +--- name: "Python CodeQL" on: workflow_dispatch: schedule: - - cron: '37 1 * * 6' + - cron: "37 1 * * 6" concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: analyze: - name: Analyze - runs-on: ubuntu-latest + name: "Analyze" + runs-on: "ubuntu-latest" permissions: - actions: read - contents: read - security-events: write + actions: "read" + contents: "read" + security-events: "write" strategy: fail-fast: false matrix: - language: [ 'python' ] + language: + - "python" steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: "Checkout repository" + uses: "actions/checkout@v6" - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + - name: "Initialize CodeQL" + uses: "github/codeql-action/init@v4" with: - languages: ${{ matrix.language }} + languages: "${{ matrix.language }}" - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: "Perform CodeQL Analysis" + uses: "github/codeql-action/analyze@v4" +... diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 75c3224d..6b57c785 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Create New Release Branch +name: "Create New Release Branch" on: workflow_dispatch: @@ -15,35 +15,33 @@ env: jobs: create-branch: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false permissions: - contents: write + contents: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install bump-my-version packaging -r plugin_template/requirements.txt - echo ::endgroup:: - name: "Setting secrets" working-directory: "pulp_python" @@ -52,8 +50,8 @@ jobs: env: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: Determine new branch name - working-directory: pulp_python + - name: "Determine new branch name" + working-directory: "pulp_python" run: | # Just to be sure... git checkout main @@ -65,42 +63,43 @@ jobs: fi echo "NEW_BRANCH=${NEW_BRANCH}" >> "$GITHUB_ENV" - - name: Create release branch - working-directory: pulp_python + - name: "Create release branch" + working-directory: "pulp_python" run: | git branch "${NEW_BRANCH}" - - name: Bump version on main branch - working-directory: pulp_python + - name: "Bump version on main branch" + working-directory: "pulp_python" run: | bump-my-version bump --no-commit minor - - name: Remove entries from CHANGES directory - working-directory: pulp_python + - name: "Remove entries from CHANGES directory" + working-directory: "pulp_python" run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + - - name: Update CI branches in template_config - working-directory: plugin_template + - name: "Update CI branches in template_config" + working-directory: "plugin_template" run: | python3 ./plugin-template pulp_python --github --latest-release-branch "${NEW_BRANCH}" git add -A - - name: Make a PR with version bump and without CHANGES/* - uses: peter-evans/create-pull-request@v6 + - name: "Make a PR with version bump and without CHANGES/*" + uses: "peter-evans/create-pull-request@v8" with: - path: pulp_python - token: ${{ secrets.RELEASE_TOKEN }} - committer: pulpbot - author: pulpbot - branch: minor-version-bump - base: main - title: Bump minor version + path: "pulp_python" + token: "${{ secrets.RELEASE_TOKEN }}" + committer: "pulpbot " + author: "pulpbot " + branch: "minor-version-bump" + base: "main" + title: "Bump minor version" commit-message: | Bump minor version delete-branch: true - - name: Push release branch - working-directory: pulp_python + - name: "Push release branch" + working-directory: "pulp_python" run: | git push origin "${NEW_BRANCH}" +... diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9e16317d..c897fcac 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,7 +13,7 @@ on: run_docs: description: "Whether to run docs jobs" required: true - type: string + type: "string" jobs: changelog: @@ -22,23 +22,22 @@ jobs: run: working-directory: "pulp_python" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.12" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install towncrier - echo ::endgroup:: - name: "Build changelog" run: | towncrier build --yes --version 4.0.0.ci docs: - if: ${{ inputs.run_docs == '1' }} - uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@main' + if: "${{ inputs.run_docs == '1' }}" + uses: "pulp/pulp-docs/.github/workflows/docs-ci.yml@main" with: - pulpdocs_ref: 'main' + pulpdocs_ref: "main" +... diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4f7181d8..bd1f718c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,30 +19,23 @@ jobs: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install -r lint_requirements.txt - echo ::endgroup:: - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows - - name: "Verify bump version config" - run: | - bump-my-version bump --dry-run release - bump-my-version show-bump - # run black separately from flake8 to get a diff - name: "Run black" run: | @@ -54,22 +47,11 @@ jobs: run: | flake8 - - name: "Run extra lint checks" - run: | - [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - - - name: "Check for any files unintentionally left out of MANIFEST.in" - run: | - check-manifest - - - name: "Verify requirements files" - run: | - python .ci/scripts/check_requirements.py - - - name: "Check for pulpcore imports outside of pulpcore.plugin" - run: | - sh .ci/scripts/check_pulpcore_imports.sh - - name: "Check for common gettext problems" run: | sh .ci/scripts/check_gettext.sh + + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh +... diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ec3fcb11..ed8149ce 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ on: schedule: # * is a special character in YAML so you have to quote this string # runs at 3:00 UTC daily - - cron: '00 3 * * *' + - cron: "00 3 * * *" workflow_dispatch: defaults: @@ -34,38 +34,46 @@ jobs: [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] changelog: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.13" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install gitpython packaging toml - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Collect changes from all branches - run: python .ci/scripts/collect_changes.py + - name: "Collect changes from all branches" + run: | + python .ci/scripts/collect_changes.py - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 + - name: "Create Pull Request" + uses: "peter-evans/create-pull-request@v8" + id: "create_pr_changelog" with: - token: ${{ secrets.RELEASE_TOKEN }} + token: "${{ secrets.RELEASE_TOKEN }}" title: "Update Changelog" body: "" branch: "changelog/update" delete-branch: true path: "pulp_python" + - name: "Mark PR automerge" + working-directory: "pulp_python" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_changelog.outputs.pull-request-number }}" + if: "steps.create_pr_changelog.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true ... diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 0e0a7936..d8054896 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -32,10 +32,10 @@ jobs: permissions: pull-requests: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Determine PR labels" @@ -43,7 +43,7 @@ jobs: pip install GitPython==3.1.42 git fetch origin ${{ github.event.pull_request.head.sha }} python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" - - uses: "actions/github-script@v7" + - uses: "actions/github-script@v8" name: "Apply PR Labels" with: script: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 525144fa..6d0d3aa8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -47,7 +47,7 @@ jobs: id-token: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" @@ -63,7 +63,7 @@ jobs: tar -xvf python-python-client.tar - name: "Publish client to pypi" - uses: pypa/gh-action-pypi-publish@release/v1 + uses: "pypa/gh-action-pypi-publish@release/v1" with: packages-dir: "pulp_python/dist/" publish-ruby-bindings: @@ -76,7 +76,7 @@ jobs: id-token: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" @@ -111,18 +111,18 @@ jobs: - "publish-ruby-bindings" permissions: - contents: write + contents: "write" env: TAG_NAME: "${{ github.ref_name }}" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" @@ -136,15 +136,15 @@ jobs: run: | # The last commit before the release commit contains the release CHANGES fragments git checkout "${TAG_NAME}~" - NOTES=$(towncrier build --draft --version $TAG_NAME) + NOTES=$(towncrier build --draft --version $TAG_NAME | .ci/scripts/clean_gh_release_notes.py pulp_python $TAG_NAME) echo "body<> $GITHUB_OUTPUT echo "$NOTES" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - name: "Create release on GitHub" - uses: "actions/github-script@v7" + uses: "actions/github-script@v8" env: - RELEASE_BODY: ${{ steps.get_release_notes.outputs.body }} + RELEASE_BODY: "${{ steps.get_release_notes.outputs.body }}" with: script: | const { TAG_NAME, RELEASE_BODY } = process.env; @@ -156,3 +156,4 @@ jobs: body: RELEASE_BODY, make_latest: "legacy", }); +... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b758bae6..e056bcfa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,21 +22,19 @@ jobs: fail-fast: false steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" token: ${{ secrets.RELEASE_TOKEN }} - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install bump-my-version towncrier - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | @@ -58,3 +56,4 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" +... diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml new file mode 100644 index 00000000..223f07cb --- /dev/null +++ b/.github/workflows/sanity.yml @@ -0,0 +1,54 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + + +# This file describes checks that should prevent a premature merge, +# but still let the tests run for demonstrations or experiments. +--- +name: "Sanity" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + sanity: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + pip install -r lint_requirements.txt + + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest + + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py + + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh +... diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 4cc50fb8..168fe4af 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -7,66 +7,126 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script prepares the scenario definition in the .ci/ansible/vars/main.yaml file. +# +# It requires the following environment: +# TEST - The name of the scenario to prepare. +# +# It may also dump the {lower,upper}bounds_constraints.txt for the specific scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -mveuo pipefail +if [ -f .github/workflows/scripts/pre_before_install.sh ]; then + source .github/workflows/scripts/pre_before_install.sh +fi -if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ] -then - BRANCH_BUILD=0 -else - BRANCH_BUILD=1 - BRANCH="${GITHUB_REF##refs/heads/}" +COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" +COMPONENT_SOURCE="./pulp_python/dist/pulp_python-${COMPONENT_VERSION}-py3-none-any.whl" +if [ "$TEST" = "s3" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" +fi +if [ "$TEST" = "azure" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]" +fi + +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u pyproject.toml > upperbounds_constraints.txt fi -if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ] -then - TAG_BUILD=0 -else - TAG_BUILD=1 - BRANCH="${GITHUB_REF##refs/tags/}" +if [[ "$TEST" = "lowerbounds" ]]; then + python3 .ci/scripts/calc_constraints.py pyproject.toml > lowerbounds_constraints.txt fi -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export COMMIT_MSG +export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") -COMPONENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") +echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" -mkdir .ci/ansible/vars || true -echo "---" > .ci/ansible/vars/main.yaml -echo "legacy_component_name: pulp_python" >> .ci/ansible/vars/main.yaml -echo "component_name: python" >> .ci/ansible/vars/main.yaml -echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml +# Compose the scenario definition. +mkdir -p .ci/ansible/vars -export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh -export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh +cat > .ci/ansible/vars/main.yaml << VARSYAML +--- +scenario: "${TEST}" +plugin_name: "pulp_python" +legacy_component_name: "pulp_python" +component_name: "python" +component_version: "${COMPONENT_VERSION}" +pulp_env: {} +pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} +pulp_scheme: "https" +api_root: "${PULP_API_ROOT}" +image: + name: "pulp" + tag: "ci_build" + ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest" + source: "${COMPONENT_SOURCE}" + ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) + upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) + lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) + webserver_snippet: $(test -f pulp_python/app/webserver_snippets/nginx.conf && echo -n true || echo -n false ) +extra_files: + - origin: "pulp_python" + destination: "pulp_python" +services: + - name: "pulp" + image: "pulp:ci_build" + volumes: + - "./settings:/etc/pulp" + - "../../../pulp-openapi-generator:/root/pulp-openapi-generator" + env: + PULP_WORKERS: "4" + PULP_HTTPS: "true" +VARSYAML -if [ -f $PRE_BEFORE_INSTALL ]; then - source $PRE_BEFORE_INSTALL +if [ "$TEST" = "s3" ]; then + MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA + MINIO_SECRET_KEY=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 + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "minio" + image: "minio/minio" + env: + MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}" + MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}" + command: "server /data" +s3_test: true +minio_access_key: "${MINIO_ACCESS_KEY}" +minio_secret_key: "${MINIO_SECRET_KEY}" +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true} +pulp_scenario_env: {} +VARSYAML fi -if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] -then - echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml +if [ "$TEST" = "azure" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-azurite" + image: "mcr.microsoft.com/azure-storage/azurite" + command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0" +azure_test: true +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "content_origin": null, "domain_enabled": true} +pulp_scenario_env: {} +VARSYAML fi -for i in {1..3} -do - ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 -done -if [[ $s -gt 0 ]] -then - echo "Failed to install amazon.aws" - exit $s +if [ "$TEST" = "gcp" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-gcp" + image: "fsouza/fake-gcs-server" + volumes: + - "storage_data:/etc/pulp" + command: " -scheme http" +gcp_test: true +pulp_scenario_settings: null +pulp_scenario_env: {} +VARSYAML fi -if [[ "$TEST" = "pulp" ]]; then - python3 .ci/scripts/calc_constraints.py -u pyproject.toml > upperbounds_constraints.txt -fi -if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_constraints.py pyproject.toml > lowerbounds_constraints.txt -fi +cat >> .ci/ansible/vars/main.yaml << VARSYAML +... +VARSYAML +cat .ci/ansible/vars/main.yaml -if [ -f $POST_BEFORE_INSTALL ]; then - source $POST_BEFORE_INSTALL +if [ -f .github/workflows/scripts/post_before_install.sh ]; then + source .github/workflows/scripts/post_before_install.sh fi diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 11404f45..13809a87 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -7,48 +7,46 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script dumps some files to help understand the setup of the test scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -euv - source .github/workflows/scripts/utils.sh -export PRE_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/pre_before_script.sh -export POST_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/post_before_script.sh - -if [[ -f $PRE_BEFORE_SCRIPT ]]; then - source $PRE_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/pre_before_script.sh ]]; then + source .github/workflows/scripts/pre_before_script.sh fi -# Developers should be able to reproduce the containers with this config -echo "CI vars:" -tail -v -n +1 .ci/ansible/vars/main.yaml - # Developers often want to know the final pulp config -echo "PULP CONFIG:" -tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json +echo +echo "# Pulp config:" +tail -v -n +1 .ci/ansible/settings/settings.* -echo "Containerfile:" +echo +echo "# Containerfile:" tail -v -n +1 .ci/ansible/Containerfile -echo "Constraints Files:" -# The need not even exist. +echo +echo "# Constraints Files:" +# They need not even exist. tail -v -n +1 ../*/*constraints.txt || true -# Needed for some functional tests -cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" -cmd_prefix bash -c "usermod -a -G wheel pulp" +echo +echo "# pip list outside the container" +pip list -if [[ "${REDIS_DISABLED:-false}" == true ]]; then - cmd_prefix bash -c "s6-rc -d change redis" - echo "The Redis service was disabled for $TEST" -fi +echo +echo "# pip list inside the container" +cmd_prefix bash -c "pip3 list" + +echo +echo "# State of the containers" +docker ps -a -if [[ -f $POST_BEFORE_SCRIPT ]]; then - source $POST_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/post_before_script.sh ]]; then + source .github/workflows/scripts/post_before_script.sh fi -# Lots of plugins try to use this path, and throw warnings if they cannot access it. -cmd_prefix mkdir /.pytest_cache -cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 29c47c5b..9df47ccd 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -7,129 +7,43 @@ # # For more info visit https://github.com/pulp/plugin_template +set -euv + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. REPO_ROOT="$PWD" -set -euv - source .github/workflows/scripts/utils.sh -PLUGIN_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" -PLUGIN_SOURCE="./pulp_python/dist/pulp_python-${PLUGIN_VERSION}-py3-none-any.whl" - -export PULP_API_ROOT="/pulp/" - PIP_REQUIREMENTS=("pulp-cli") # This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} +if [[ "$TEST" = "s3" ]]; then +for i in {1..3} +do + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 +done +if [[ $s -gt 0 ]] +then + echo "Failed to install amazon.aws" + exit $s +fi +fi + # Check out the pulp-cli branch matching the installed version. PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli cd .ci/ansible/ -if [ "$TEST" = "s3" ]; then - PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[s3]" -fi -if [ "$TEST" = "azure" ]; then - PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[azure]" -fi - -cat >> vars/main.yaml << VARSYAML -image: - name: pulp - tag: "ci_build" -plugins: - - name: pulp_python - source: "${PLUGIN_SOURCE}" -VARSYAML -if [[ -f ../../ci_requirements.txt ]]; then - cat >> vars/main.yaml << VARSYAML - ci_requirements: true -VARSYAML -fi -if [ "$TEST" = "pulp" ]; then - cat >> vars/main.yaml << VARSYAML - upperbounds: true -VARSYAML -fi -if [ "$TEST" = "lowerbounds" ]; then - cat >> vars/main.yaml << VARSYAML - lowerbounds: true -VARSYAML -fi - -cat >> vars/main.yaml << VARSYAML -services: - - name: pulp - image: "pulp:ci_build" - volumes: - - ./settings:/etc/pulp - - ./ssh:/keys/ - - ~/.config:/var/lib/pulp/.config - - ../../../pulp-openapi-generator:/root/pulp-openapi-generator - env: - PULP_WORKERS: "4" - PULP_HTTPS: "true" -VARSYAML - -cat >> vars/main.yaml << VARSYAML -pulp_env: {} -pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} -pulp_scheme: https -pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest -VARSYAML - -if [ "$TEST" = "s3" ]; then - export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA - export MINIO_SECRET_KEY=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 - sed -i -e '/^services:/a \ - - name: minio\ - image: minio/minio\ - env:\ - MINIO_ACCESS_KEY: "'$MINIO_ACCESS_KEY'"\ - MINIO_SECRET_KEY: "'$MINIO_SECRET_KEY'"\ - command: "server /data"' vars/main.yaml - sed -i -e '$a s3_test: true\ -minio_access_key: "'$MINIO_ACCESS_KEY'"\ -minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true}\ -pulp_scenario_env: {}\ -' vars/main.yaml - export PULP_API_ROOT="/rerouted/djnd/" -fi - -if [ "$TEST" = "azure" ]; then - sed -i -e '/^services:/a \ - - name: ci-azurite\ - image: mcr.microsoft.com/azure-storage/azurite\ - volumes:\ - - ./azurite:/etc/pulp\ - command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml - sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "content_origin": null, "domain_enabled": true}\ -pulp_scenario_env: {}\ -' vars/main.yaml -fi - -echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" - -if [ "${PULP_API_ROOT:-}" ]; then - sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml -fi -pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" --username "admin" --password "password" +pulp config create --base-url https://pulp --api-root "${PULP_API_ROOT}" --username "admin" --password "password" cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml ansible-playbook start_container.yaml -# .config needs to be accessible by the pulp user in the container, but some -# files will likely be modified on the host by post/pre scripts. -chmod 777 ~/.config/pulp_smash/ -chmod 666 ~/.config/pulp_smash/settings.json # Plugins often write to ~/.config/pulp/cli.toml from the host chmod 777 ~/.config/pulp chmod 666 ~/.config/pulp/cli.toml @@ -155,6 +69,10 @@ if [[ "$TEST" = "azure" ]]; then az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING fi -echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list" -echo ::endgroup:: +# Needed for some functional tests +cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" +cmd_prefix bash -c "usermod -a -G wheel pulp" + +# Lots of plugins try to use this path, and throw warnings if they cannot access it. +cmd_prefix mkdir /.pytest_cache +cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88cfd8f3..fbddb788 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ on: inputs: matrix_env: required: true - type: string + type: "string" defaults: run: @@ -24,38 +24,38 @@ jobs: strategy: fail-fast: false matrix: - env: ${{ fromJSON(inputs.matrix_env) }} + env: "${{ fromJSON(inputs.matrix_env) }}" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Download plugin package" - uses: "actions/download-artifact@v4" + uses: "actions/download-artifact@v8" with: name: "plugin_package" path: "pulp_python/dist/" - name: "Download API specs" - uses: "actions/download-artifact@v4" + uses: "actions/download-artifact@v8" with: name: "api_spec" path: "pulp_python/" - name: "Download client packages" - uses: "actions/download-artifact@v4" + uses: "actions/download-artifact@v8" with: name: "python-client.tar" path: "pulp_python" @@ -70,16 +70,14 @@ jobs: - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install build towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - name: "Set environment variables" run: | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - name: "Before Install" + - name: "Prepare Scenario Definition" run: | .github/workflows/scripts/before_install.sh shell: "bash" @@ -99,7 +97,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - name: "Before Script" + - name: "Dump CI Metadata" run: | .github/workflows/scripts/before_script.sh shell: "bash" @@ -108,7 +106,6 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" - name: "Script" run: | @@ -125,7 +122,7 @@ jobs: docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt - name: "Upload Deprecations" - uses: actions/upload-artifact@v4 + uses: "actions/upload-artifact@v5" with: name: "deprecations-${{ matrix.env.TEST }}" path: "pulp_python/deprecations-${{ matrix.env.TEST }}.txt" diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index 4565da8c..39320dca 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -19,7 +19,7 @@ jobs: update_backport_labels: runs-on: "ubuntu-latest" steps: - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Configure Git with pulpbot name and email" @@ -28,12 +28,11 @@ jobs: git config --global user.email 'pulp-infra@redhat.com' - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install requests pyyaml - echo ::endgroup:: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" - name: "Update labels" run: | python3 .github/workflows/scripts/update_backport_labels.py env: GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" +... diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 1d1e1968..7254881a 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -12,7 +12,7 @@ on: schedule: # * is a special character in YAML so you have to quote this string # runs at 2:30 UTC every Sunday - - cron: '30 2 * * 0' + - cron: "30 2 * * 0" workflow_dispatch: jobs: @@ -23,27 +23,25 @@ jobs: fail-fast: false steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install gitpython packaging -r plugin_template/requirements.txt - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -55,7 +53,7 @@ jobs: ../plugin_template/scripts/update_ci.sh - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_main" with: token: "${{ secrets.RELEASE_TOKEN }}" @@ -74,7 +72,7 @@ jobs: env: GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" continue-on-error: true - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -86,7 +84,7 @@ jobs: ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_3_11" with: token: "${{ secrets.RELEASE_TOKEN }}" @@ -105,7 +103,7 @@ jobs: env: GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" continue-on-error: true - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -117,7 +115,7 @@ jobs: ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_3_12" with: token: "${{ secrets.RELEASE_TOKEN }}" @@ -136,7 +134,7 @@ jobs: env: GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" continue-on-error: true - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -148,7 +146,7 @@ jobs: ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_3_13" with: token: "${{ secrets.RELEASE_TOKEN }}" @@ -167,7 +165,7 @@ jobs: env: GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" continue-on-error: true - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -179,7 +177,7 @@ jobs: ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_3_25" with: token: "${{ secrets.RELEASE_TOKEN }}" diff --git a/CHANGES.md b/CHANGES.md index 67775803..13480dd9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,28 @@ [//]: # (towncrier release notes start) +## 3.26.1 (2026-03-10) {: #3.26.1 } + +#### Bugfixes {: #3.26.1-bugfix } + +- Fixed sync issue where non https URLs were not allowed. + +--- + +## 3.26.0 (2026-02-26) {: #3.26.0 } + +#### Features {: #3.26.0-feature } + +- Added support for recreating and fixing metadata files to `repair_metadata` endpoint. + [#1099](https://github.com/pulp/pulp_python/issues/1099) + +#### Bugfixes {: #3.26.0-bugfix } + +- Fixed edge case where metadata file did not match wheel metadata. + [#1101](https://github.com/pulp/pulp_python/issues/1101) + +--- + ## 3.25.1 (2026-02-16) {: #3.25.1 } #### Bugfixes {: #3.25.1-bugfix } diff --git a/CHANGES/1099.feature b/CHANGES/1099.feature deleted file mode 100644 index 1699af0e..00000000 --- a/CHANGES/1099.feature +++ /dev/null @@ -1 +0,0 @@ -Added support for recreating and fixing metadata files to `repair_metadata` endpoint. diff --git a/CHANGES/1101.bugfix b/CHANGES/1101.bugfix deleted file mode 100644 index 0bfcf6d5..00000000 --- a/CHANGES/1101.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed edge case where metadata file did not match wheel metadata. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index b15c8ce8..2d024a00 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -10,7 +10,7 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.26.0.dev" + version = "3.26.2.dev" python_package_name = "pulp-python" domain_compatible = True diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index b364c3dd..1ceaf207 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -65,7 +65,6 @@ def create_bandersnatch_config(remote): config = BandersnatchConfig() config["mirror"]["master"] = remote.url config["mirror"]["workers"] = str(remote.download_concurrency) - config["mirror"]["allow_non_https"] = "true" if not config.has_section("plugins"): config.add_section("plugins") config["plugins"]["enabled"] = "blocklist_release\n" @@ -120,7 +119,7 @@ async def run(self): if not isinstance(downloader, HttpDownloader): raise ValueError("Only HTTP(S) is supported for python syncing") - async with Master(url) as master: + async with Master(url, allow_non_https=True) as master: # Replace the session with the remote's downloader session old_session = master.session master.session = downloader.session diff --git a/pyproject.toml b/pyproject.toml index cdbc7bec..87e67285 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = 'setuptools.build_meta' [project] name = "pulp-python" -version = "3.26.0.dev" +version = "3.26.2.dev" description = "pulp-python plugin for the Pulp Project" readme = "README.md" authors = [ @@ -79,7 +79,7 @@ ignore = [ [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.26.0.dev" +current_version = "3.26.2.dev" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" diff --git a/template_config.yml b/template_config.yml index c9bea811..9ee9dfaf 100644 --- a/template_config.yml +++ b/template_config.yml @@ -100,7 +100,6 @@ test_deprecations: true test_gcp: false test_lowerbounds: true test_performance: false -test_reroute: true test_s3: true use_issue_template: true ...