From eaa3c5d37e291c765bca460a9d776edd02ec5545 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 8 Nov 2023 20:58:38 +0000 Subject: [PATCH 01/84] Add changelog for 3.11.0 [noissue] --- CHANGES.rst | 16 ++++++++++++++++ CHANGES/+pulpcore-3.40.feature | 1 - CHANGES/579.feature | 1 - 3 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 CHANGES/+pulpcore-3.40.feature delete mode 100644 CHANGES/579.feature diff --git a/CHANGES.rst b/CHANGES.rst index 33b23e83..a6026153 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,22 @@ Changelog .. towncrier release notes start +3.11.0 (2023-11-08) +=================== + + +Features +-------- + +- Added pulpcore 3.40 compatibility. + `#40 `__ +- Added import export support of python content. + `#579 `__ + + +---- + + 3.10.0 (2023-05-17) =================== diff --git a/CHANGES/+pulpcore-3.40.feature b/CHANGES/+pulpcore-3.40.feature deleted file mode 100644 index 7aa25a62..00000000 --- a/CHANGES/+pulpcore-3.40.feature +++ /dev/null @@ -1 +0,0 @@ -Added pulpcore 3.40 compatibility. diff --git a/CHANGES/579.feature b/CHANGES/579.feature deleted file mode 100644 index 19390c76..00000000 --- a/CHANGES/579.feature +++ /dev/null @@ -1 +0,0 @@ -Added import export support of python content. From d95177a98a1f1540cabe400119b63a92de08cb3c Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 8 Nov 2023 20:58:39 +0000 Subject: [PATCH 02/84] Release 3.11.0 GH Issues: 579 [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulp_python/app/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c4084695..288fe22d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.11.0.dev +current_version = 3.11.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 17fcf22e..09b1c9a9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.11.0.dev" +version = "3.11.0" # The full version, including alpha/beta/rc tags. -release = "3.11.0.dev" +release = "3.11.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index b5227fa8..486a2265 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.0.dev" + version = "3.11.0" python_package_name = "pulp-python" diff --git a/setup.py b/setup.py index 078afa7e..2eeb14eb 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.0.dev", + version="3.11.0", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From bb686070c37d972627ed15dbb7e53d9c1d42fe31 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 8 Nov 2023 20:58:39 +0000 Subject: [PATCH 03/84] Bump to 3.11.1.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulp_python/app/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 288fe22d..d0ce3af6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.11.0 +current_version = 3.11.1.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 09b1c9a9..1be52760 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.11.0" +version = "3.11.1.dev" # The full version, including alpha/beta/rc tags. -release = "3.11.0" +release = "3.11.1.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 486a2265..576b80ab 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.0" + version = "3.11.1.dev" python_package_name = "pulp-python" diff --git a/setup.py b/setup.py index 2eeb14eb..16287f31 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.0", + version="3.11.1.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From 1558481cafdd76bf51cd0e8fea78ed539779ed81 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 9 Nov 2023 16:24:22 -0500 Subject: [PATCH 04/84] Fix 3.11.0 changelog entry [noissue] --- CHANGES.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a6026153..bb2afffc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -21,7 +21,6 @@ Features -------- - Added pulpcore 3.40 compatibility. - `#40 `__ - Added import export support of python content. `#579 `__ From f4ad1753c23c770bc52477f9b899ad58d14d2916 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 11 Apr 2024 20:33:35 +0000 Subject: [PATCH 05/84] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 22 +- .ci/ansible/start_container.yaml | 24 +- .ci/scripts/check_release.py | 57 +++- .ci/scripts/collect_changes.py | 17 +- .github/template_gitref | 2 +- .github/workflows/build.yml | 44 +++ .github/workflows/changelog.yml | 32 +- .github/workflows/ci.yml | 250 ++++---------- .github/workflows/codeql-analysis.yml | 16 +- .github/workflows/create-branch.yml | 36 +- .github/workflows/kanban.yml | 4 +- .github/workflows/lint.yml | 56 +++ .github/workflows/nightly.yml | 286 +++------------- .github/workflows/pr_checks.yml | 62 ++++ .github/workflows/publish.yml | 322 ++++++++++++++++++ .github/workflows/release.yml | 274 ++------------- .github/workflows/scripts/before_install.sh | 14 +- .github/workflows/scripts/check_commit.sh | 13 +- .github/workflows/scripts/install.sh | 36 +- .../scripts/install_python_client.sh | 68 ++-- .../workflows/scripts/install_ruby_client.sh | 45 ++- .../workflows/scripts/publish_client_gem.sh | 25 +- .../workflows/scripts/publish_client_pypi.sh | 24 +- .github/workflows/scripts/publish_docs.sh | 31 +- .../workflows/scripts/publish_plugin_pypi.sh | 28 +- .../scripts/push_branch_and_tag_to_github.sh | 25 +- .github/workflows/scripts/release.sh | 26 ++ .github/workflows/scripts/script.sh | 152 +++++---- .../scripts/update_backport_labels.py | 8 +- .github/workflows/test.yml | 170 +++++++++ .github/workflows/update-labels.yml | 22 +- .github/workflows/update_ci.yml | 49 ++- CHANGES/.TEMPLATE.rst | 14 +- docs/_scripts/base.sh | 2 + functest_requirements.txt | 2 +- lint_requirements.txt | 2 +- template_config.yml | 30 +- unittest_requirements.txt | 2 +- 38 files changed, 1235 insertions(+), 1057 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pr_checks.yml create mode 100644 .github/workflows/publish.yml create mode 100755 .github/workflows/scripts/release.sh create mode 100644 .github/workflows/test.yml diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 0eb5d5cc..338ff697 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,28 +1,22 @@ -FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }} +FROM {{ ci_base | default(pulp_default_container) }} # Add source directories to container {% for item in plugins %} -{% if item.source.startswith("./") or item.ci_requirements | default(false) %} ADD ./{{ item.name }} ./{{ item.name }} -{% endif %} {% endfor %} # Install python packages +# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads +# Hacking botocore (https://github.com/boto/botocore/pull/1990) RUN pip3 install -{%- for item in plugins -%} -{%- if item.name == "pulp-certguard" -%} -{{ " " }}python-dateutil rhsm -{%- endif -%} -{{ " " }}{{ item.source }} -{%- if item.name == "pulpcore" -%} {%- if s3_test | default(false) -%} -[s3] -{%- elif azure_test | default(false) -%} -[azure] -{%- elif gcp_test | default(false) -%} -[google] +{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} +{%- for item in plugins -%} +{{ " " }}{{ item.source }} +{%- if item.lowerbounds | default(false) -%} +{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt {%- endif -%} {%- if item.ci_requirements | default(false) -%} {{ " " }}-r ./{{ item.name }}/ci_requirements.txt diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index b19a2054..47e5221e 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -83,22 +83,14 @@ command: "docker logs pulp" failed_when: true - - block: - - name: "Check version of component being tested" - assert: - that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver) - fail_msg: | - Component {{ component_name }} was expected to be installed in version {{ component_version }}. - Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}. - rescue: - - name: "Check version of component being tested (legacy)" - assert: - that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] | canonical_semver == (component_version | canonical_semver) - fail_msg: | - Component {{ legacy_component_name }} was expected to be installed in version {{ component_version }}. - Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] }}. + - name: "Check version of component being tested" + assert: + that: + - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver) + fail_msg: | + Component {{ item.app_label }} was expected to be installed in version {{ component_version }}. + Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}. + loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}" - name: "Set pulp password in .netrc" copy: diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 70b48a2a..92ffd685 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -29,7 +29,7 @@ def main(): "--branches", default="supported", help="A comma separated list of branches to check for releases. Can also use keyword: " - "'supported'. Defaults to 'supported', see `ci_update_branches` in " + "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) opts = parser.parse_args() @@ -46,12 +46,15 @@ def main(): if branches == "supported": with open(f"{d}/template_config.yml", mode="r") as f: tc = yaml.safe_load(f) - branches = tc["ci_update_branches"] - branches.append(DEFAULT_BRANCH) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) else: - branches = branches.split(",") + branches = set(branches.split(",")) - if diff := set(branches) - set(available_branches): + if diff := branches - set(available_branches): print(f"Supplied branches contains non-existent branches! {diff}") exit(1) @@ -62,7 +65,7 @@ def main(): if branch != DEFAULT_BRANCH: # Check if a Z release is needed changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") - z_release = False + z_changelog = False for change in changes.split("\n"): # Check each changelog file to make sure everything checks out _, ext = os.path.splitext(change) @@ -72,18 +75,48 @@ def main(): f"{branch} release branch!" ) elif ext in Z_CHANGELOG_EXTS: - z_release = True - if z_release: + z_changelog = True + + last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}") + req_txt_diff = repo.git.diff( + f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" + ) + if z_changelog or req_txt_diff: # Blobless clone does not have file contents for Z branches, # check commit message for last Z bump git_branch = f"origin/{branch}" - next_version = repo.git.log( - "--oneline", "--grep=Bump to", "-n 1", git_branch, "--", ".bumpversion.cfg" - ).split("to")[-1] + next_version = None + bump_commit = repo.git.log( + "--oneline", + "--grep=Bump version", + "-n 1", + git_branch, + "--", + ".bumpversion.cfg", + ) + if bump_commit: + next_version = bump_commit.split("→ ")[-1] + # If not found - try old-commit-msg + if not next_version: + bump_commit = repo.git.log( + "--oneline", + "--grep=Bump to", + "-n 1", + git_branch, + "--", + ".bumpversion.cfg", + ) + next_version = bump_commit.split("to ")[-1] if bump_commit else None + + # You could, theoretically, be next_vers==None here - but that's always + # been true for this script. next_version = Version(next_version) + reason = "CHANGES" if z_changelog else "requirements.txt" print( f"A Z-release is needed for {branch}, " - f"New Version: {next_version.base_version}" + f"Prev: {last_tag}, " + f"Next: {next_version.base_version}, " + f"Reason: {reason}" ) releases.append(next_version) else: diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 2141ebc5..49153880 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -11,7 +11,7 @@ import toml from git import GitCommandError, Repo -from pkg_resources import parse_version +from packaging.version import parse as parse_version # Read Towncrier settings tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"] @@ -19,15 +19,21 @@ CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( "start_string", - "\n" - if CHANGELOG_FILE.endswith(".md") - else ".. towncrier release notes start\n", + ( + "\n" + if CHANGELOG_FILE.endswith(".md") + else ".. towncrier release notes start\n" + ), ) TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})") +# Build a regex to find the header of a changelog section. +# It must have a single capture group to single out the version. +# see help(re.split) for more info. NAME_REGEX = r".*" -VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)" +VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*" +VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})" DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" TITLE_REGEX = ( "(" @@ -35,6 +41,7 @@ TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX") ) .replace("NAME_REGEX", NAME_REGEX) + .replace("VERSION_REGEX", VERSION_CAPTURE_REGEX, 1) .replace("VERSION_REGEX", VERSION_REGEX) .replace("DATE_REGEX", DATE_REGEX) + ")" diff --git a/.github/template_gitref b/.github/template_gitref index cd7b2753..9ec4e6fe 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-250-gf4315cf +2021.08.26-326-ge5addc7 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..7f9252f5 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,44 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Build" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + build: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install packaging twine wheel + echo ::endgroup:: + - name: "Build package" + run: | + python3 setup.py sdist bdist_wheel --python-tag py3 + twine check dist/* + - name: "Upload Package whl" + uses: "actions/upload-artifact@v3" + with: + name: "plugin_package" + path: "pulp_python/dist/" + if-no-files-found: "error" + retention-days: 5 diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 621ff732..38a60302 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -6,53 +6,53 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Python changelog update +name: "Python changelog update" on: push: branches: - - main + - "main" paths: - - CHANGES.rst - - CHANGES.md + - "CHANGES.rst" + - "CHANGES.md" workflow_dispatch: jobs: update-changelog: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS pip install -r doc_requirements.txt echo ::endgroup:: - - name: Fake api schema + - name: "Fake api schema" run: | mkdir -p docs/_build/html echo "{}" > docs/_build/html/api.json mkdir -p docs/_static echo "{}" > docs/_static/api.json - - name: + - name: "Build Docs" run: | - pip install "Jinja2<3.1" make diagrams html - working-directory: ./docs + working-directory: "./docs" env: PULP_CONTENT_ORIGIN: "http://localhost/" - - name: Publish changlog to pulpproject.org - run: .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} + - name: "Publish changlog to pulpproject.org" + run: | + .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} env: - PULP_DOCS_KEY: ${{ secrets.PULP_DOCS_KEY }} + PULP_DOCS_KEY: "${{ secrets.PULP_DOCS_KEY }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12b8db8b..422fa133 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,218 +6,88 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Python CI +name: "Python CI" on: {pull_request: {branches: ['*']}} concurrency: group: ${{ github.ref_name }}-${{ github.workflow }} cancel-in-progress: true -jobs: +defaults: + run: + working-directory: "pulp_python" - ready-to-ship: - runs-on: ubuntu-latest +jobs: + check-commits: + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 0 - - uses: actions/setup-python@v4 + path: "pulp_python" + - uses: "actions/setup-python@v4" with: - python-version: "3.8" - - name: Install requirements - run: pip3 install github - - name: Check commit message + python-version: "3.11" + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install requests pygithub + echo ::endgroup:: + - name: "Check commit message" if: github.event_name == 'pull_request' env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - run: sh .github/workflows/scripts/check_commit.sh - - name: Verify requirements files - run: python .ci/scripts/check_requirements.py - single_commit: - runs-on: ubuntu-latest - name: Assert single commit - if: github.base_ref == 'main' - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Checkout main - run: git fetch origin main - - name: create local main branch - run: git branch main origin/main - - name: Commit Count Check - run: test `git log --oneline --no-merges HEAD ^main | wc -l ` = 1 - - lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - # lint_requirements contains tools needed for flake8, etc. - - name: Install requirements - run: pip3 install -r lint_requirements.txt - - - - # Lint code. - - name: Run flake8 - run: flake8 - - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" - - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest - - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh - - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh - - test: - runs-on: ubuntu-latest - # run only after lint finishes - needs: lint - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - TEST: lowerbounds - outputs: - deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} - deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} - deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} - deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install httpie + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables + .github/workflows/scripts/check_commit.sh + - name: "Verify requirements files" run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + python .ci/scripts/check_requirements.py - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - if: github.event_name != 'pull_request' - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} + lint: + uses: "./.github/workflows/lint.yml" - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + build: + needs: "lint" + uses: "./.github/workflows/build.yml" - - name: Extract Deprecations from Logs - id: deprecations - run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT + test: + needs: "build" + uses: "./.github/workflows/test.yml" - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" deprecations: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" if: github.base_ref == 'main' - needs: test + needs: "test" steps: - - name: Fail on deprecations + - name: "Create working directory" run: | - test -z "${{ needs.test.outputs.deprecations-pulp }}" - test -z "${{ needs.test.outputs.deprecations-azure }}" - test -z "${{ needs.test.outputs.deprecations-s3 }}" - test -z "${{ needs.test.outputs.deprecations-lowerbounds }}" - - name: Print deprecations - if: failure() + mkdir -p "pulp_python" + working-directory: "." + - name: "Download Deprecations" + uses: actions/download-artifact@v3 + with: + name: "deprecations" + path: "pulp_python" + - name: "Print deprecations" run: | - echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-lowerbounds }}" | base64 -d - + cat deprecations-*.txt | sort -u + ! cat deprecations-*.txt | grep '[^[:space:]]' + ready-to-ship: + # This is a dummy dependent task to have a single entry for the branch protection rules. + runs-on: "ubuntu-latest" + needs: + - "check-commits" + - "lint" + - "test" + if: "always()" + steps: + - name: "Collect needed jobs results" + working-directory: "." + run: | + echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result' + echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0' + echo "CI says: Looks good!" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1dff2755..899147c4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,13 +30,13 @@ jobs: language: [ 'python' ] steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 19a5faad..d5f7ebca 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -21,26 +21,27 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 0 - path: pulp_python + path: "pulp_python" - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version jinja2 pyyaml + pip install bump2version jinja2 pyyaml packaging echo ::endgroup:: - - name: Setting secrets - working-directory: pulp_python - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + - name: "Setting secrets" + working-directory: "pulp_python" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} + SECRETS_CONTEXT: "${{ toJson(secrets) }}" - name: Determine new branch name working-directory: pulp_python @@ -70,8 +71,21 @@ jobs: run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + + - name: Checkout plugin template + uses: actions/checkout@v4 + with: + repository: pulp/plugin_template + path: plugin_template + fetch-depth: 0 + + - name: Update CI branches in template_config + working-directory: plugin_template + run: | + python3 ./plugin-template pulp_python --github --latest-release-branch "${NEW_BRANCH}" + git add -A + - name: Make a PR with version bump and without CHANGES/* - uses: peter-evans/create-pull-request@v4 + uses: peter-evans/create-pull-request@v6 with: path: pulp_python token: ${{ secrets.RELEASE_TOKEN }} diff --git a/.github/workflows/kanban.yml b/.github/workflows/kanban.yml index 97e75008..370f4bee 100644 --- a/.github/workflows/kanban.yml +++ b/.github/workflows/kanban.yml @@ -58,10 +58,10 @@ jobs: runs-on: ubuntu-latest name: Find issues linked to a PR outputs: - linked-issues: ${{ steps.linked-issues.outputs.issues }} + linked-issues: ${{ steps.linked-issues.outputs.issues }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Get Linked Issues Action uses: kin/gh-action-get-linked-issues@v1.0 id: linked-issues diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..43aee3b1 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,56 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Lint" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install -r lint_requirements.txt + echo ::endgroup:: + + - name: Lint workflow files + run: | + yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + + # Lint code. + - name: Run flake8 + run: flake8 + + - name: Run extra lint checks + run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + + # check for any files unintentionally left out of MANIFEST.in + - name: Check manifest + run: check-manifest + + - name: Check for pulpcore imports outside of pulpcore.plugin + run: sh .ci/scripts/check_pulpcore_imports.sh + + - name: Check for gettext problems + run: sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 66862def..1cc7dec2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Python Nightly CI +name: "Python Nightly CI" on: schedule: # * is a special character in YAML so you have to quote this string @@ -14,175 +14,41 @@ on: - cron: '00 3 * * *' workflow_dispatch: +defaults: + run: + working-directory: "pulp_python" + concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: - test: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - - TEST: generate-bindings - - TEST: lowerbounds - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - uses: ruby/setup-ruby@v1 - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - with: - ruby-version: "2.6" - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + build: + uses: "./.github/workflows/build.yml" - - name: Upload python client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client.tar - path: python-client.tar - - - name: Upload python client docs - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client-docs.tar - path: python-client-docs.tar - - - name: Upload ruby client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: ruby-client.tar - path: ruby-client.tar - - name: Upload built docs - if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v3 - with: - name: docs.tar - path: docs/docs.tar - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + test: + needs: "build" + uses: "./.github/workflows/test.yml" changelog: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 0 + path: "pulp_python" - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install gitpython toml + pip install gitpython packaging toml echo ::endgroup:: - - name: Configure Git with pulpbot name and email + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' @@ -191,129 +57,63 @@ jobs: run: python .ci/scripts/collect_changes.py - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 + uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.RELEASE_TOKEN }} - title: 'Update Changelog' - body: '' - branch: 'changelog/update' + title: "Update Changelog" + body: "" + branch: "changelog/update" delete-branch: true + path: "pulp_python" publish: runs-on: ubuntu-latest needs: test - env: - TEST: publish - steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 1 + path: "pulp_python" - - uses: actions/setup-python@v4 + - uses: actions/download-artifact@v3 with: - python-version: "3.8" + name: "plugin_package" + path: "pulp_python/dist/" - - uses: ruby/setup-ruby@v1 + - uses: "actions/setup-python@v4" with: - ruby-version: "2.6" + python-version: "3.11" - - name: Install httpie + - name: "Install python dependencies" run: | - echo ::group::HTTPIE - pip install httpie + echo ::group::PYDEPS + pip install requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1' echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - name: Set environment variables + - name: "Set environment variables" run: | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - name: Install python dependencies - run: | - echo ::group::PYDEPS - pip install wheel - echo ::endgroup:: - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Download built docs uses: actions/download-artifact@v3 with: - name: docs.tar + name: "docs.tar" + path: "pulp_python" - name: Download Python client docs uses: actions/download-artifact@v3 with: - name: python-client-docs.tar + name: "python-client-docs.tar" + path: "pulp_python" + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" - name: Publish docs to pulpproject.org run: | tar -xvf docs.tar -C ./docs .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} - - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml new file mode 100644 index 00000000..f2a44ab6 --- /dev/null +++ b/.github/workflows/pr_checks.yml @@ -0,0 +1,62 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: Python PR static checks +on: + pull_request_target: + types: [opened, synchronize, reopened] + +# This workflow runs with elevated permissions. +# Do not even think about running a single bit of code from the PR. +# Static analysis should be fine however. + +concurrency: + group: ${{ github.event.pull_request.number }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + single_commit: + runs-on: ubuntu-latest + name: Label multiple commit PR + permissions: + pull-requests: write + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + - name: Commit Count Check + run: | + git fetch origin ${{ github.event.pull_request.head.sha }} + echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV" + - uses: actions/github-script@v7 + with: + script: | + const labelName = "multi-commit"; + const { COMMIT_COUNT } = process.env; + + if (COMMIT_COUNT == 1) + { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } + } + else + { + await github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: [labelName], + }); + } diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..e1715162 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,322 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Python Publish Release" +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + +defaults: + run: + working-directory: "pulp_python" + +jobs: + build: + uses: "./.github/workflows/build.yml" + + build-bindings-docs: + needs: + - "build" + runs-on: "ubuntu-latest" + # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests + env: + TEST: "docs" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "pulp_python/dist/" + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV + echo ::endgroup:: + + # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to + # setup the Pulp instance. + - name: "Before Install" + run: | + .github/workflows/scripts/before_install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Install" + run: | + .github/workflows/scripts/install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Install Python client" + run: | + .github/workflows/scripts/install_python_client.sh + shell: "bash" + - name: "Install Ruby client" + run: | + .github/workflows/scripts/install_ruby_client.sh + shell: "bash" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v3" + with: + name: "python-client.tar" + path: "pulp_python/python-python-client.tar" + if-no-files-found: "error" + + - name: "Upload python client docs" + uses: "actions/upload-artifact@v3" + with: + name: "python-client-docs.tar" + path: "pulp_python/python-python-client-docs.tar" + if-no-files-found: "error" + - name: "Upload ruby client packages" + uses: "actions/upload-artifact@v3" + with: + name: "ruby-client.tar" + path: "pulp_python/python-ruby-client.tar" + if-no-files-found: "error" + - name: Build docs + run: | + export DJANGO_SETTINGS_MODULE=pulpcore.app.settings + export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py + make -C docs/ PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" diagrams html + tar -cvf docs/docs.tar docs/_build + + - name: "Upload built docs" + uses: actions/upload-artifact@v3 + with: + name: "docs.tar" + path: "pulp_python/docs/docs.tar" + + - name: "Logs" + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + publish-package: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "pulp_python/dist/" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Deploy plugin to pypi" + run: | + .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.ref_name }} + publish-python-bindings: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - name: "Download Python client" + uses: "actions/download-artifact@v3" + with: + name: "python-client.tar" + path: "pulp_python/" + + - name: "Untar python client packages" + run: | + tar -xvf python-python-client.tar + + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Publish client to pypi" + run: | + bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.ref_name }} + publish-ruby-bindings: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - name: "Download Ruby client" + uses: "actions/download-artifact@v3" + with: + name: "ruby-client.tar" + path: "pulp_python/" + + - name: "Untar Ruby client packages" + run: | + tar -xvf python-ruby-client.tar + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Publish client to rubygems" + run: | + bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} + publish-docs: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install 'packaging~=21.3' requests + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Download built docs" + uses: "actions/download-artifact@v3" + with: + name: "docs.tar" + path: "pulp_python/" + + - name: "Download Python client docs" + uses: "actions/download-artifact@v3" + with: + name: "python-client-docs.tar" + path: "pulp_python/" + + - name: "Publish docs to pulpproject.org" + run: | + tar -xvf docs.tar + .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }} + + create-gh-release: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + - "publish-package" + - "publish-python-bindings" + - "publish-ruby-bindings" + - "publish-docs" + + steps: + - name: "Create release on GitHub" + uses: "actions/github-script@v7" + env: + TAG_NAME: "${{ github.ref_name }}" + with: + script: | + const { TAG_NAME } = process.env; + + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: TAG_NAME, + make_latest: "legacy", + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 85224a88..ff0adfb3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,282 +9,52 @@ name: Python Release Pipeline on: workflow_dispatch: - inputs: - release: - description: "Release tag (e.g. 3.2.1)" - before_script: - description: | - Bash code to run before bindings and docs are built. This should only be used when re-running - a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst - required: false -env: - RELEASE_WORKFLOW: true +defaults: + run: + working-directory: "pulp_python" jobs: build-artifacts: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 0 + path: "pulp_python" + token: ${{ secrets.RELEASE_TOKEN }} - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging~=21.3 bump2version gitpython towncrier==19.9.0 wheel requests + pip install bump2version towncrier echo ::endgroup:: - - name: Configure Git with pulpbot name and email + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Create the release commit, tag it, create a post-release commit, and build plugin package - run: python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} - - - name: 'Tar files' - run: tar -cvf pulp_python.tar . - - - name: 'Upload Artifact' - uses: actions/upload-artifact@v3 - with: - name: pulp_python.tar - path: pulp_python.tar - - - build-bindings-docs: - needs: build-artifacts - runs-on: ubuntu-latest - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: docs - - steps: - - uses: actions/download-artifact@v3 - with: - name: pulp_python.tar - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: Untar repository - run: | - tar -xf pulp_python.tar - - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - - name: Install httpie + - name: "Setting secrets" run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: | - export PLUGIN_VERSION=${{ github.event.inputs.release }} - .github/workflows/scripts/install.sh + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - shell: bash - - - name: Additional before_script - run: ${{ github.event.inputs.before_script }} - shell: bash - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - name: Install Ruby client - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Upload python client packages - uses: actions/upload-artifact@v3 - with: - name: python-client.tar - path: python-client.tar - - - name: Upload python client docs - uses: actions/upload-artifact@v3 - with: - name: python-client-docs.tar - path: python-client-docs.tar - - name: Upload ruby client packages - uses: actions/upload-artifact@v3 - with: - name: ruby-client.tar - path: ruby-client.tar - - name: Build docs - run: | - export DJANGO_SETTINGS_MODULE=pulpcore.app.settings - export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py - make -C docs/ PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" diagrams html - tar -cvf docs/docs.tar docs/_build - - - name: Upload built docs - uses: actions/upload-artifact@v3 - with: - name: docs.tar - path: docs/docs.tar - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - + SECRETS_CONTEXT: "${{ toJson(secrets) }}" - publish: - runs-on: ubuntu-latest - needs: build-bindings-docs - - env: - TEST: publish - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - steps: - - uses: actions/download-artifact@v3 - with: - name: pulp_python.tar - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: Configure Git with pulpbot name and email + - name: "Tag the release" run: | - git config --global user.name 'pulpbot' - git config --global user.email 'pulp-infra@redhat.com' - - - name: Untar repository - run: | - tar -xf pulp_python.tar - - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + .github/workflows/scripts/release.sh + shell: "bash" env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install python dependencies - run: | - echo ::group::PYDEPS - pip install gitpython requests packaging~=21.3 tweepy - echo ::endgroup:: - - - name: Push branch and tag to GitHub - run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - - name: Download built docs - uses: actions/download-artifact@v3 - with: - name: docs.tar - - - name: Download Python client docs - uses: actions/download-artifact@v3 - with: - name: python-client-docs.tar - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }} - - name: Deploy plugin to pypi - run: bash .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} - - name: Download Python client - uses: actions/download-artifact@v3 - with: - name: python-client.tar - - - name: Untar python client packages - run: tar -xvf python-client.tar - - - name: Publish client to pypi - run: bash .github/workflows/scripts/publish_client_pypi.sh - - name: Download Ruby client - uses: actions/download-artifact@v3 - with: - name: ruby-client.tar - - - name: Untar Ruby client packages - run: tar -xvf ruby-client.tar - - - name: Publish client to rubygems - run: bash .github/workflows/scripts/publish_client_gem.sh - - - name: Update GitHub - continue-on-error: true - run: | - set -euv - export COMMIT_MSG=$(git log --format=%B --no-merges -1) - export GH_ISSUES=$(echo $COMMIT_MSG | grep -o "GH Issues: .*" | awk '{print $3}') - pip install pygithub - - echo "GH Issues $GH_ISSUES" - python .ci/scripts/update_github.py - - - name: Create release on GitHub - run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 474720c1..5abc9c98 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -55,14 +55,6 @@ then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml fi - -cd .. - -git clone --depth=1 https://github.com/pulp/pulp-openapi-generator.git - -# Intall requirements for ansible playbooks -pip install docker netaddr boto3 ansible - for i in {1..3} do ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3 @@ -73,11 +65,9 @@ then exit $s fi -cd pulp_python - if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_requirements.txt - mv lowerbounds_requirements.txt requirements.txt + python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt + sed -i 's/\[.*\]//g' lowerbounds_constraints.txt fi if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index 7780e8be..e6b6d5e3 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -8,20 +8,13 @@ # For more info visit https://github.com/pulp/plugin_template # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." set -euv -echo ::group::REQUESTS -pip3 install requests - -pip3 install pygithub - -echo ::endgroup:: - -for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') +for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') do - python3 .ci/scripts/validate_commit_message.py $sha + python3 .ci/scripts/validate_commit_message.py "$SHA" VALUE=$? if [ "$VALUE" -gt 0 ]; then exit $VALUE diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 292a3ed5..6c9fdd4a 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,6 +15,9 @@ set -euv source .github/workflows/scripts/utils.sh +PLUGIN_VERSION="$(sed -n -e 's/^\s*current_version\s*=\s*//p' .bumpversion.cfg | python -c 'from packaging.version import Version; print(Version(input()))')" +PLUGIN_SOURCE="./pulp_python/dist/pulp_python-${PLUGIN_VERSION}-py3-none-any.whl" + export PULP_API_ROOT="/pulp/" PIP_REQUIREMENTS=("pulp-cli") @@ -34,25 +37,31 @@ then fi cd .ci/ansible/ - -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - PLUGIN_NAME=./pulp_python/dist/pulp_python-$PLUGIN_VERSION-py3-none-any.whl -else - PLUGIN_NAME=./pulp_python +if [ "$TEST" = "s3" ]; then + PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[s3]" fi +if [ "$TEST" = "azure" ]; then + PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[azure]" +fi + cat >> vars/main.yaml << VARSYAML image: name: pulp tag: "ci_build" plugins: - name: pulp_python - source: "${PLUGIN_NAME}" + source: "${PLUGIN_SOURCE}" VARSYAML if [[ -f ../../ci_requirements.txt ]]; then cat >> vars/main.yaml << VARSYAML ci_requirements: true VARSYAML fi +if [ "$TEST" = "lowerbounds" ]; then + cat >> vars/main.yaml << VARSYAML + lowerbounds: true +VARSYAML +fi cat >> vars/main.yaml << VARSYAML services: @@ -72,9 +81,7 @@ cat >> vars/main.yaml << VARSYAML pulp_env: {} pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} pulp_scheme: https - -pulp_container_tag: "latest" - +pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest VARSYAML if [ "$TEST" = "s3" ]; then @@ -121,7 +128,7 @@ if [ "${PULP_API_ROOT:-}" ]; then sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml fi -pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" +pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" --username "admin" --password "password" if [[ "$TEST" != "docs" ]]; then cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" fi @@ -156,10 +163,17 @@ cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null sudo update-ca-certificates echo ::endgroup:: +# Add our azcert.crt certificate to the container image along with the certificates from certifi +# so that we can use HTTPS with our fake Azure CI. certifi is self-contained and doesn't allow +# extension or modification of the trust store, so we do a weird and hacky thing (above) where we just +# overwrite or append to certifi's trust store behind it's back. +# +# We do this for both the CI host and the CI image. if [[ "$TEST" = "azure" ]]; then AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())') + PULPCERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /usr/local/lib/python3.8/site-packages/certifi/cacert.pem > /dev/null + cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a "$PULPCERTIFI" > /dev/null cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /etc/pki/tls/cert.pem > /dev/null AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://ci-azurite:10000/devstoreaccount1;' az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index 553c78cd..8c1fe8f0 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -9,45 +9,61 @@ set -mveuo pipefail -export PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" - # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -pip install twine wheel +source .github/workflows/scripts/utils.sh -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi +PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" +export PULP_URL +PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" +export PULP_API_ROOT -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) +REPORTED_STATUS="$(pulp status)" +REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "python" -r '.versions[] | select(.component == $plugin) | .version')" +VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" -if [ "$response" == "200" ]; +pushd ../pulp-openapi-generator +rm -rf pulp_python-client + +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" then - echo "pulp_python client $VERSION has already been released. Installing from PyPI." - docker exec pulp pip3 install pulp-python-client==$VERSION - mkdir -p dist - tar cvf python-client.tar ./dist - exit + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=python" + USE_LOCAL_API_JSON=1 ./generate.sh pulp_python python "$VERSION" +else + ./generate.sh pulp_python python "$VERSION" fi -cd ../pulp-openapi-generator -rm -rf pulp_python-client -./generate.sh pulp_python python $VERSION -cd pulp_python-client +pushd pulp_python-client python setup.py sdist bdist_wheel --python-tag py3 -find . -name "*.whl" -exec docker exec pulp pip3 install /root/pulp-openapi-generator/pulp_python-client/{} \; -tar cvf ../../pulp_python/python-client.tar ./dist + +twine check "dist/pulp_python_client-$VERSION-py3-none-any.whl" +twine check "dist/pulp_python-client-$VERSION.tar.gz" + +cmd_prefix pip3 install "/root/pulp-openapi-generator/pulp_python-client/dist/pulp_python_client-${VERSION}-py3-none-any.whl" +tar cvf ../../pulp_python/python-python-client.tar ./dist find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; find ./docs/* -exec sed -i 's/README//g' {} \; cp README.md docs/index.md sed -i 's/docs\///g' docs/index.md find ./docs/* -exec sed -i 's/\.md//g' {} \; -tar cvf ../../pulp_python/python-client-docs.tar ./docs -exit $? + +cat >> mkdocs.yml << DOCSYAML +--- +site_name: PulpPython Client +site_description: Python bindings +site_author: Pulp Team +site_url: https://docs.pulpproject.org/pulp_python_client/ +repo_name: pulp/pulp_python +repo_url: https://github.com/pulp/pulp_python +theme: readthedocs +DOCSYAML + +# Building the bindings docs +mkdocs build + +# Pack the built site. +tar cvf ../../pulp_python/python-python-client-docs.tar ./site +popd +popd diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh index 1dd84089..7ab198a5 100755 --- a/.github/workflows/scripts/install_ruby_client.sh +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -7,37 +7,36 @@ # # For more info visit https://github.com/pulp/plugin_template -set -euv +set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -export PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" +source .github/workflows/scripts/utils.sh -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi +PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" +export PULP_URL +PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" +export PULP_API_ROOT + +REPORTED_STATUS="$(pulp status)" +REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "python" -r '.versions[] | select(.component == $plugin) | .version')" +VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) +pushd ../pulp-openapi-generator +rm -rf pulp_python-client -if [ "$response" == "200" ]; +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" then - echo "pulp_python client $VERSION has already been released. Installing from RubyGems.org." - gem install pulp_python_client -v $VERSION - touch pulp_python_client-$VERSION.gem - tar cvf ruby-client.tar ./pulp_python_client-$VERSION.gem - exit + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=python" + USE_LOCAL_API_JSON=1 ./generate.sh pulp_python ruby "$VERSION" +else + ./generate.sh pulp_python ruby "$VERSION" fi -cd ../pulp-openapi-generator -rm -rf pulp_python-client -./generate.sh pulp_python ruby $VERSION -cd pulp_python-client +pushd pulp_python-client gem build pulp_python_client -gem install --both ./pulp_python_client-$VERSION.gem -tar cvf ../../pulp_python/ruby-client.tar ./pulp_python_client-$VERSION.gem +gem install --both "./pulp_python_client-$VERSION.gem" +tar cvf ../../pulp_python/python-ruby-client.tar "./pulp_python_client-$VERSION.gem" +popd +popd diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh index 2cd1d052..bd367010 100755 --- a/.github/workflows/scripts/publish_client_gem.sh +++ b/.github/workflows/scripts/publish_client_gem.sh @@ -12,27 +12,24 @@ set -euv # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. - -mkdir ~/.gem || true -touch ~/.gem/credentials -echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials -sudo chmod 600 ~/.gem/credentials - -export VERSION=$(ls pulp_python_client* | sed -rn 's/pulp_python_client-(.*)\.gem/\1/p') +VERSION="$1" if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit + echo "No version specified." + exit 1 fi -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) +RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://rubygems.org/gems/pulp_python_client/versions/$VERSION")" -if [ "$response" == "200" ]; +if [ "$RESPONSE" == "200" ]; then echo "pulp_python client $VERSION has already been released. Skipping." exit fi -GEM_FILE="$(ls pulp_python_client-*)" -gem push ${GEM_FILE} +mkdir -p ~/.gem +touch ~/.gem/credentials +echo "--- +:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials +sudo chmod 600 ~/.gem/credentials +gem push "pulp_python_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index 83dbcaf6..544c760d 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -10,28 +10,24 @@ set -euv # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." -pip install twine - -export VERSION=$(ls dist | sed -rn 's/pulp_python-client-(.*)\.tar.gz/\1/p') +VERSION="$1" if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit + echo "No version specified." + exit 1 fi -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) +RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-python-client/$VERSION/")" -if [ "$response" == "200" ]; +if [ "$RESPONSE" == "200" ]; then echo "pulp_python client $VERSION has already been released. Skipping." exit fi -twine check dist/pulp_python_client-$VERSION-py3-none-any.whl || exit 1 -twine check dist/pulp_python-client-$VERSION.tar.gz || exit 1 -twine upload dist/pulp_python_client-$VERSION-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/pulp_python-client-$VERSION.tar.gz -u pulp -p $PYPI_PASSWORD - -exit $? +twine upload -u __token__ -p "$PYPI_API_TOKEN" \ +"dist/pulp_python_client-$VERSION-py3-none-any.whl" \ +"dist/pulp_python-client-$VERSION.tar.gz" \ +; diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh index b82378f2..db944810 100755 --- a/.github/workflows/scripts/publish_docs.sh +++ b/.github/workflows/scripts/publish_docs.sh @@ -10,7 +10,7 @@ set -euv # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." mkdir ~/.ssh touch ~/.ssh/pulp-infra @@ -20,44 +20,29 @@ echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts chmod 644 ~/.ssh/known_hosts -pip3 install packaging - export PYTHONUNBUFFERED=1 export DJANGO_SETTINGS_MODULE=pulpcore.app.settings export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py export WORKSPACE=$PWD -eval "$(ssh-agent -s)" #start the ssh agent +# start the ssh agent +eval "$(ssh-agent -s)" ssh-add ~/.ssh/pulp-infra -python3 .github/workflows/scripts/docs-publisher.py --build-type $1 --branch $2 +python3 .github/workflows/scripts/docs-publisher.py --build-type "$1" --branch "$2" if [[ "$GITHUB_WORKFLOW" == "Python changelog update" ]]; then # Do not build bindings docs on changelog update exit fi -pip install mkdocs pymdown-extensions "Jinja2<3.1" - -mkdir -p ../bindings -tar -xvf python-client-docs.tar --directory ../bindings -cd ../bindings -cat >> mkdocs.yml << DOCSYAML ---- -site_name: PulpPython Client -site_description: Python bindings -site_author: Pulp Team -site_url: https://docs.pulpproject.org/pulp_python_client/ -repo_name: pulp/pulp_python -repo_url: https://github.com/pulp/pulp_python -theme: readthedocs -DOCSYAML - -# Building the bindings docs -mkdocs build +mkdir -p ../python-bindings +tar -xvf python-python-client-docs.tar --directory ../python-bindings +pushd ../python-bindings # publish to docs.pulpproject.org/pulp_python_client rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/ # publish to docs.pulpproject.org/pulp_python_client/en/{release} rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/en/"$2" +popd diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index 981f7707..1dd7d883 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -7,23 +7,27 @@ # # For more info visit https://github.com/pulp/plugin_template +set -euv + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -euv +VERSION="$1" -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$1/) -if [ "$response" == "200" ]; -then - echo "pulp_python $1 has already been released. Skipping." - exit +if [[ -z "$VERSION" ]]; then + echo "No version specified." + exit 1 fi -pip install twine +RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-python/$VERSION/")" -twine check dist/pulp_python-$1-py3-none-any.whl || exit 1 -twine check dist/pulp-python-$1.tar.gz || exit 1 -twine upload dist/pulp_python-$1-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/pulp-python-$1.tar.gz -u pulp -p $PYPI_PASSWORD +if [ "$RESPONSE" == "200" ]; +then + echo "pulp_python $VERSION has already been released. Skipping." + exit +fi -exit $? +twine upload -u __token__ -p "$PYPI_API_TOKEN" \ +"dist/pulp_python-$VERSION-py3-none-any.whl" \ +"dist/pulp-python-$VERSION.tar.gz" \ +; diff --git a/.github/workflows/scripts/push_branch_and_tag_to_github.sh b/.github/workflows/scripts/push_branch_and_tag_to_github.sh index 2f13c78d..421eb171 100755 --- a/.github/workflows/scripts/push_branch_and_tag_to_github.sh +++ b/.github/workflows/scripts/push_branch_and_tag_to_github.sh @@ -1,21 +1,16 @@ #!/bin/sh -set -e -BRANCH_NAME=$(echo $GITHUB_REF | sed -rn 's/refs\/heads\/(.*)/\1/p') +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template -ref_string=$(git show-ref --tags | grep refs/tags/$1) +set -eu -SHA=${ref_string:0:40} +BRANCH_NAME="$(echo "$GITHUB_REF" | sed -rn 's/refs\/heads\/(.*)/\1/p')" -remote_repo=https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git +remote_repo="https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" -git push "${remote_repo}" $BRANCH_NAME - -curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/git/refs \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "ref": "refs/tags/$1", - "sha": "$SHA" -} -EOF +git push "${remote_repo}" "$BRANCH_NAME" "$1" diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh new file mode 100755 index 00000000..9525f229 --- /dev/null +++ b/.github/workflows/scripts/release.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eu -o pipefail + +BRANCH=$(git branch --show-current) + +if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]] +then + echo ERROR: This is not a release branch! + exit 1 +fi + +NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +echo "Release ${NEW_VERSION}" + +if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] +then + echo ERROR: Version does not match release branch + exit 1 +fi + +towncrier build --yes --version "${NEW_VERSION}" +bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump2version patch --commit + +git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 42ae3d1f..131ad0dc 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -12,7 +12,6 @@ set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -REPO_ROOT="$PWD" source .github/workflows/scripts/utils.sh @@ -30,112 +29,135 @@ export PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" if [[ "$TEST" = "docs" ]]; then if [[ "$GITHUB_WORKFLOW" == "Python CI" ]]; then - pip install towncrier==19.9.0 - towncrier --yes --version 4.0.0.ci + towncrier build --yes --version 4.0.0.ci fi + # Legacy Docs Build cd docs make PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24PULP_URL" diagrams html tar -cvf docs.tar ./_build cd .. - if [ -f $POST_DOCS_TEST ]; then - source $POST_DOCS_TEST + if [ -f "$POST_DOCS_TEST" ]; then + source "$POST_DOCS_TEST" fi exit fi REPORTED_STATUS="$(pulp status)" -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - REPORTED_VERSION="$(echo $REPORTED_STATUS | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')" - response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$REPORTED_VERSION/) - if [ "$response" == "200" ]; - then - echo "pulp_python $REPORTED_VERSION has already been released. Skipping running tests." - exit - fi -fi - echo "machine pulp login admin password password " | cmd_user_stdin_prefix bash -c "cat >> ~pulp/.netrc" # Some commands like ansible-galaxy specifically require 600 -cmd_user_stdin_prefix bash -c "chmod 600 ~pulp/.netrc" +cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -cd ../pulp-openapi-generator -if [ "$(echo "$REPORTED_STATUS" | jq -r '.versions[0].package')" = "null" ] +# Generate and install binding +pushd ../pulp-openapi-generator +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" then - # We are on an old version of pulpcore without package in the status report - for app_label in $(echo "$REPORTED_STATUS" | jq -r '.versions[].component') - do - if [ "$app_label" = "core" ] - then - item=pulpcore - else - item="pulp_${app_label}" - fi - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done + # Use app_label to generate api.json and package to produce the proper package name. + + if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] + then + # Workaround: Domains are not supported by the published bindings. + # Generate new bindings for all packages. + for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") + do + echo $item + COMPONENT="$(jq -r '.component' <<<"$item")" + VERSION="$(jq -r '.version' <<<"$item")" + MODULE="$(jq -r '.module' <<<"$item")" + PACKAGE="${MODULE%%.*}" + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" + USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" + cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" + sudo rm -rf "./${PACKAGE}-client" + done + else + # Sadly: Different pulpcore-versions aren't either... + for item in $(jq -r '.versions[]| select(.component!="python")| tojson' <<<"$REPORTED_STATUS") + do + echo $item + COMPONENT="$(jq -r '.component' <<<"$item")" + VERSION="$(jq -r '.version' <<<"$item")" + MODULE="$(jq -r '.module' <<<"$item")" + PACKAGE="${MODULE%%.*}" + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" + USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" + cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" + sudo rm -rf "./${PACKAGE}-client" + done + fi else - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[].package|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done -fi + # Infer the client name from the package name by replacing "-" with "_". + # Use the component to infer the package name on older versions of pulpcore. -cd $REPO_ROOT + if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] + then + # Workaround: Domains are not supported by the published bindings. + # Generate new bindings for all packages. + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') + do + ./generate.sh "${item}" python + cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" + sudo rm -rf "./${item}-client" + done + else + # Sadly: Different pulpcore-versions aren't either... + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="python")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') + do + ./generate.sh "${item}" python + cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" + sudo rm -rf "./${item}-client" + done + fi +fi +popd -cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" -cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt +# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') -cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt | tee -a "$CERTIFI" > /dev/null" +cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" # check for any uncommitted migrations echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations python --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs pulp_python.tests.unit" - +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --pyargs pulp_python.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_python.tests.performance" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_python.tests.performance" else - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_python.tests.performance.test_$PERFORMANCE_TEST" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_python.tests.performance.test_${PERFORMANCE_TEST}" fi exit fi -if [ -f $FUNC_TEST_SCRIPT ]; then - source $FUNC_TEST_SCRIPT +if [ -f "$FUNC_TEST_SCRIPT" ]; then + source "$FUNC_TEST_SCRIPT" else - - if [[ "$GITHUB_WORKFLOW" == "Python Nightly CI/CD" ]] || [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" - - - else - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_python.tests.functional -m 'not parallel'" - - - fi - + if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] + then + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" + else + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel'" + fi fi pushd ../pulp-cli pip install -r test_requirements.txt -pytest -v -m pulp_python +pytest -v -m "pulp_python" popd -if [ -f $POST_SCRIPT ]; then - source $POST_SCRIPT +if [ -f "$POST_SCRIPT" ]; then + source "$POST_SCRIPT" fi diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py index a6e24116..967984a4 100755 --- a/.github/workflows/scripts/update_backport_labels.py +++ b/.github/workflows/scripts/update_backport_labels.py @@ -32,10 +32,14 @@ def random_color(): assert response.status_code == 200 old_labels = set([x["name"] for x in response.json() if x["name"].startswith("backport-")]) -# get ci_update_branches from template_config.yml +# get list of branches from template_config.yml with open("./template_config.yml", "r") as f: plugin_template = yaml.safe_load(f) -new_labels = set(["backport-" + x for x in plugin_template["ci_update_branches"]]) +branches = set(plugin_template["supported_release_branches"]) +latest_release_branch = plugin_template["latest_release_branch"] +if latest_release_branch is not None: + branches.add(latest_release_branch) +new_labels = {"backport-" + x for x in branches} # delete old labels that are not in new labels for label in old_labels.difference(new_labels): diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..e5b52a60 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,170 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Test" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + test: + runs-on: "ubuntu-latest" + strategy: + fail-fast: false + matrix: + env: + - TEST: pulp + - TEST: docs + - TEST: azure + - TEST: s3 + - TEST: lowerbounds + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "pulp_python/dist/" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV + echo ::endgroup:: + + - name: "Set environment variables" + run: | + echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + + - name: "Before Install" + run: | + .github/workflows/scripts/before_install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - uses: ruby/setup-ruby@v1 + if: ${{ env.TEST == 'pulp' }} + with: + ruby-version: "2.6" + + - name: "Install" + run: | + .github/workflows/scripts/install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Before Script" + run: | + .github/workflows/scripts/before_script.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" + + - name: "Install Python client" + run: | + .github/workflows/scripts/install_python_client.sh + shell: "bash" + - name: "Install Ruby client" + if: "${{ env.TEST == 'pulp' }}" + run: | + .github/workflows/scripts/install_ruby_client.sh + shell: "bash" + + - name: "Script" + run: | + .github/workflows/scripts/script.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Extract Deprecations from Logs" + run: | + docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt + + - name: "Upload Deprecations" + uses: actions/upload-artifact@v3 + with: + name: "deprecations" + path: "pulp_python/deprecations-${{ matrix.env.TEST }}.txt" + if-no-files-found: "error" + retention-days: 5 + - name: Upload python client packages + if: ${{ env.TEST == 'pulp' }} + uses: actions/upload-artifact@v3 + with: + name: "python-client.tar" + path: "pulp_python/python-python-client.tar" + if-no-files-found: "error" + retention-days: 5 + + - name: Upload python client docs + if: ${{ env.TEST == 'pulp' }} + uses: actions/upload-artifact@v3 + with: + name: "python-client-docs.tar" + path: "pulp_python/python-python-client-docs.tar" + if-no-files-found: "error" + retention-days: 5 + - name: Upload Ruby client + if: ${{ env.TEST == 'pulp' }} + uses: actions/upload-artifact@v3 + with: + name: "ruby-client.tar" + path: "pulp_python/python-ruby-client.tar" + if-no-files-found: "error" + retention-days: 5 + - name: Upload built docs + if: ${{ env.TEST == 'docs' }} + uses: actions/upload-artifact@v3 + with: + name: "docs.tar" + path: "pulp_python/docs/docs.tar" + + - name: "Logs" + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index 1dd998a2..d67136cb 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -7,33 +7,33 @@ --- -name: Python Update Labels +name: "Python Update Labels" on: push: branches: - - main + - "main" paths: - - 'template_config.yml' + - "template_config.yml" jobs: update_backport_labels: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.8" - - name: Configure Git with pulpbot name and email + python-version: "3.11" + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS pip install requests pyyaml echo ::endgroup:: - - uses: actions/checkout@v3 - - name: Update labels + - uses: "actions/checkout@v4" + - name: "Update labels" run: | python3 .github/workflows/scripts/update_backport_labels.py env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 91560ce3..87951824 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -7,65 +7,64 @@ --- -name: Python CI Update +name: "Python CI Update" on: schedule: # * is a special character in YAML so you have to quote this string # runs at 2:30 UTC every Sunday - cron: '30 2 * * 0' - workflow_dispatch: jobs: update: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: "actions/checkout@v4" with: - repository: pulp/plugin_template - path: plugin_template fetch-depth: 0 + repository: "pulp/plugin_template" + path: "plugin_template" - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS pip install gitpython requests packaging jinja2 pyyaml echo ::endgroup:: - - name: Configure Git with pulpbot name and email + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - uses: actions/checkout@v3 + - uses: "actions/checkout@v4" with: - path: pulp_python - ref: 'main' fetch-depth: 0 + path: "pulp_python" + ref: "main" - - name: Run update - working-directory: pulp_python + - name: "Run update" + working-directory: "pulp_python" run: | ../plugin_template/scripts/update_ci.sh - - name: Create Pull Request for CI files - uses: peter-evans/create-pull-request@v4 + - name: "Create Pull Request for CI files" + uses: "peter-evans/create-pull-request@v6" with: - token: ${{ secrets.RELEASE_TOKEN }} - path: pulp_python - committer: pulpbot - author: pulpbot - title: 'Update CI files for branch main' - body: '[noissue]' - branch: 'update-ci/main' - base: 'main' + token: "${{ secrets.RELEASE_TOKEN }}" + path: "pulp_python" + committer: "pulpbot " + author: "pulpbot " + title: "Update CI files for branch main" + body: "" + branch: "update-ci/main" + base: "main" commit-message: | Update CI files diff --git a/CHANGES/.TEMPLATE.rst b/CHANGES/.TEMPLATE.rst index ab3826e7..49c2305d 100644 --- a/CHANGES/.TEMPLATE.rst +++ b/CHANGES/.TEMPLATE.rst @@ -1,5 +1,12 @@ - -{# TOWNCRIER TEMPLATE #} +{% if render_title %} +{% if versiondata.name %} +{{ versiondata.name }} {{ versiondata.version }} ({{ versiondata.date }}) +{{ top_underline * ((versiondata.name + versiondata.version + versiondata.date)|length + 4)}} +{% else %} +{{ versiondata.version }} ({{ versiondata.date }}) +{{ top_underline * ((versiondata.version + versiondata.date)|length + 3)}} +{% endif %} +{% endif %} {% for section, _ in sections.items() %} {% set underline = underlines[0] %}{% if section %}{{section}} {{ underline * section|length }}{% set underline = underlines[1] %} @@ -35,3 +42,6 @@ No significant changes. {% endif %} {% endfor %} ---- + + + diff --git a/docs/_scripts/base.sh b/docs/_scripts/base.sh index b80b5d10..a1b45744 100755 --- a/docs/_scripts/base.sh +++ b/docs/_scripts/base.sh @@ -25,5 +25,7 @@ if [ ! -f ~/.config/pulp/settings.toml ]; then base_url = "$BASE_ADDR" verify_ssl = false format = "json" +username = "admin" +password = "password" EOF fi diff --git a/functest_requirements.txt b/functest_requirements.txt index c43d8b3d..17cfad8f 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -1,5 +1,5 @@ git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash -pytest +pytest<8 lxml twine pypi-simple diff --git a/lint_requirements.txt b/lint_requirements.txt index 4e7958f1..2422f937 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -8,4 +8,4 @@ # python packages handy for developers, but not required by pulp check-manifest flake8 - +yamllint diff --git a/template_config.yml b/template_config.yml index 308a2a42..32c874a1 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,20 +1,18 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-248-g4bfc3e1 +# generated with plugin_template@2021.08.26-326-ge5addc7 -additional_repos: [] api_root: /pulp/ black: false check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true +ci_base_image: ghcr.io/pulp/pulp-ci-centos9 ci_env: {} ci_trigger: '{pull_request: {branches: [''*'']}}' -ci_update_branches: [] ci_update_docs: false -ci_update_release_behavior: null cli_package: pulp-cli cli_repo: https://github.com/pulp/pulp-cli.git core_import_allowed: [] @@ -31,19 +29,16 @@ flake8_ignore: [] github_org: pulp issue_tracker: github kanban: true +latest_release_branch: null lint_requirements: true noissue_marker: '[noissue]' parallel_test_workers: 8 plugin_app_label: python -plugin_camel: PulpPython -plugin_camel_short: Python -plugin_caps: PULP_PYTHON -plugin_caps_short: PYTHON -plugin_dash: pulp-python -plugin_dash_short: python plugin_default_branch: main plugin_name: pulp_python -plugin_snake: pulp_python +plugins: +- app_label: python + name: pulp_python post_job_template: null pre_job_template: null publish_docs_to_pulpprojectdotorg: true @@ -53,25 +48,21 @@ pulp_env_gcp: {} pulp_env_s3: {} pulp_scheme: https pulp_settings: - orphan_protection_time: 0 - pypi_api_hostname: https://pulp:443 allowed_export_paths: /tmp allowed_import_paths: /tmp + orphan_protection_time: 0 + pypi_api_hostname: https://pulp:443 pulp_settings_azure: null pulp_settings_gcp: null pulp_settings_s3: null -pulpprojectdotorg_key_id: null pydocstyle: true -pypi_username: pulp -python_version: '3.8' release_email: pulp-infra@redhat.com release_user: pulpbot -run_pulpcore_tests_for_plugins: false -single_commit_check: true stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 stalebot_limit_to_pulls: true +supported_release_branches: [] sync_ci: true test_azure: true test_cli: true @@ -81,6 +72,7 @@ test_lowerbounds: true test_performance: false test_reroute: true test_s3: true -update_github: true use_issue_template: true +use_legacy_docs: true +use_unified_docs: false diff --git a/unittest_requirements.txt b/unittest_requirements.txt index 93253de9..f5b662af 100644 --- a/unittest_requirements.txt +++ b/unittest_requirements.txt @@ -1,2 +1,2 @@ mock -pytest +pytest<8 From e9495efb7323c5135009b53fda155f19b8f52f70 Mon Sep 17 00:00:00 2001 From: Gerrod Date: Tue, 19 Mar 2024 08:04:53 -0400 Subject: [PATCH 06/84] Merge pull request #651 from gerrod3/fake-netrc Remove 'missing netrc' log messages when syncing (cherry picked from commit 9befa2383ff63099f9dcfed3d21f7ce304fceff1) --- pulp_python/app/tasks/sync.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index c4f93dfa..5238c69e 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -1,9 +1,10 @@ import logging +import tempfile from aiohttp import ClientResponseError, ClientError from lxml.etree import LxmlError from gettext import gettext as _ -from os import environ, path +from os import environ from rest_framework import serializers @@ -111,9 +112,10 @@ async def run(self): """ If includes is specified, then only sync those,else try to sync all other packages """ - # Prevent bandersnatch from reading actual .netrc file, set to nonexistent file + # Prevent bandersnatch from reading actual .netrc file, set to empty file # See discussion on https://github.com/pulp/pulp_python/issues/581 - environ["NETRC"] = f"{path.curdir}/.fake-netrc" + fake_netrc = tempfile.NamedTemporaryFile(dir=".", delete=False) + environ["NETRC"] = fake_netrc.name # TODO Change Bandersnatch internal API to take proxy settings in from config parameters if proxy_url := self.remote.proxy_url: if self.remote.proxy_username or self.remote.proxy_password: From c636d002e0b01545a73e2880ef90810f27b99539 Mon Sep 17 00:00:00 2001 From: Gerrod Date: Thu, 11 Apr 2024 15:58:20 -0400 Subject: [PATCH 07/84] Merge pull request #656 from gerrod3/tls-false Fix sync with tls_validation=False (cherry picked from commit 6644673ff04a3ad5775f2e4995edf18e054e0758) --- CHANGES/653.bugfix | 1 + pulp_python/app/tasks/sync.py | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 CHANGES/653.bugfix diff --git a/CHANGES/653.bugfix b/CHANGES/653.bugfix new file mode 100644 index 00000000..6040d64f --- /dev/null +++ b/CHANGES/653.bugfix @@ -0,0 +1 @@ +Fixed tls_validation not being disabled when set to false on the remote. diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 5238c69e..3af26a59 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -1,6 +1,8 @@ import logging import tempfile +from typing import Optional, Any, AsyncGenerator +import aiohttp from aiohttp import ClientResponseError, ClientError from lxml.etree import LxmlError from gettext import gettext as _ @@ -128,7 +130,7 @@ async def run(self): # Bandersnatch includes leading slash when forming API urls url = self.remote.url.rstrip("/") # local & global timeouts defaults to 10secs and 5 hours - async with Master(url) as master: + async with PulpMaster(url, tls=self.remote.tls_validation) as master: deferred_download = self.remote.policy != Remote.IMMEDIATE workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY async with ProgressReport( @@ -150,6 +152,25 @@ async def run(self): await pmirror.synchronize(packages_to_sync) +class PulpMaster(Master): + """ + Pulp Master Class for Pulp specific overrides + """ + + def __init__(self, *args, tls=True, **kwargs): + self.tls = tls + super().__init__(*args, **kwargs) + + async def get( + self, path: str, required_serial: Optional[int], **kw: Any + ) -> AsyncGenerator[aiohttp.ClientResponse, None]: + """Support tls=false""" + if not self.tls: + kw["ssl"] = False + async for r in super().get(path, required_serial, **kw): + yield r + + class PulpMirror(Mirror): """ Pulp Mirror Class to perform syncing using Bandersnatch @@ -259,4 +280,4 @@ def on_error(self, exception, **kwargs): TODO This should have some error checking """ - pass + logger.error("Sync encountered an error: ", exc_info=exception) From 202b64e065ac458759ac60857b1b41c3cfbe3ff2 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 11 Apr 2024 21:28:29 +0000 Subject: [PATCH 08/84] Release 3.11.1 --- .bumpversion.cfg | 2 +- CHANGES.rst | 13 +++++++++++++ CHANGES/653.bugfix | 1 - docs/conf.py | 4 ++-- pulp_python/app/__init__.py | 2 +- setup.py | 2 +- 6 files changed, 18 insertions(+), 6 deletions(-) delete mode 100644 CHANGES/653.bugfix diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d0ce3af6..0804a7e3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.11.1.dev +current_version = 3.11.1 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/CHANGES.rst b/CHANGES.rst index bb2afffc..1e38e617 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,19 @@ Changelog .. towncrier release notes start +3.11.1 (2024-04-11) +=================== + +Bugfixes +-------- + +- Fixed tls_validation not being disabled when set to false on the remote. + `#653 `__ + + +---- + + 3.11.0 (2023-11-08) =================== diff --git a/CHANGES/653.bugfix b/CHANGES/653.bugfix deleted file mode 100644 index 6040d64f..00000000 --- a/CHANGES/653.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed tls_validation not being disabled when set to false on the remote. diff --git a/docs/conf.py b/docs/conf.py index 1be52760..accfc276 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.11.1.dev" +version = "3.11.1" # The full version, including alpha/beta/rc tags. -release = "3.11.1.dev" +release = "3.11.1" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 576b80ab..4f089f12 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.1.dev" + version = "3.11.1" python_package_name = "pulp-python" diff --git a/setup.py b/setup.py index 16287f31..a14d11f8 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.1.dev", + version="3.11.1", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From ec7533e57a7c47e3b1accd2494594db67f1ffbc6 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 11 Apr 2024 21:28:29 +0000 Subject: [PATCH 09/84] =?UTF-8?q?Bump=20version:=203.11.1=20=E2=86=92=203.?= =?UTF-8?q?11.2.dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulp_python/app/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0804a7e3..ade32719 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.11.1 +current_version = 3.11.2.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index accfc276..5aa1fb18 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.11.1" +version = "3.11.2.dev" # The full version, including alpha/beta/rc tags. -release = "3.11.1" +release = "3.11.2.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 4f089f12..0e370115 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.1" + version = "3.11.2.dev" python_package_name = "pulp-python" diff --git a/setup.py b/setup.py index a14d11f8..168fb600 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.1", + version="3.11.2.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From 733863bffff408d4f5580675370562820f6d8988 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 21 Apr 2024 02:34:28 +0000 Subject: [PATCH 10/84] Update CI files [noissue] --- .ci/scripts/check_release.py | 34 +++++----------------------------- .github/template_gitref | 2 +- 2 files changed, 6 insertions(+), 30 deletions(-) diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 92ffd685..4f2f3172 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -82,35 +82,11 @@ def main(): f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" ) if z_changelog or req_txt_diff: - # Blobless clone does not have file contents for Z branches, - # check commit message for last Z bump - git_branch = f"origin/{branch}" - next_version = None - bump_commit = repo.git.log( - "--oneline", - "--grep=Bump version", - "-n 1", - git_branch, - "--", - ".bumpversion.cfg", - ) - if bump_commit: - next_version = bump_commit.split("→ ")[-1] - # If not found - try old-commit-msg - if not next_version: - bump_commit = repo.git.log( - "--oneline", - "--grep=Bump to", - "-n 1", - git_branch, - "--", - ".bumpversion.cfg", - ) - next_version = bump_commit.split("to ")[-1] if bump_commit else None - - # You could, theoretically, be next_vers==None here - but that's always - # been true for this script. - next_version = Version(next_version) + curr_version = Version(last_tag) + assert curr_version.base_version.startswith( + branch + ), "Current-version has to belong to the current branch!" + next_version = Version(f"{branch}.{curr_version.micro + 1}") reason = "CHANGES" if z_changelog else "requirements.txt" print( f"A Z-release is needed for {branch}, " diff --git a/.github/template_gitref b/.github/template_gitref index 9ec4e6fe..418b21cd 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-326-ge5addc7 +2021.08.26-327-g6a0d984 From b64d4a76dbfdf6bf7d25687fbbcf21a0543312ce Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 12 May 2024 02:36:08 +0000 Subject: [PATCH 11/84] Update CI files [noissue] --- .github/template_gitref | 2 +- .github/workflows/build.yml | 4 ++-- .github/workflows/changelog.yml | 2 +- .github/workflows/ci.yml | 7 ++++--- .github/workflows/create-branch.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/nightly.yml | 10 +++++----- .github/workflows/publish.yml | 28 ++++++++++++++-------------- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 16 ++++++++-------- .github/workflows/update-labels.yml | 2 +- .github/workflows/update_ci.yml | 2 +- 12 files changed, 40 insertions(+), 39 deletions(-) diff --git a/.github/template_gitref b/.github/template_gitref index 418b21cd..2232d657 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-327-g6a0d984 +2021.08.26-330-g0b7d0c1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f9252f5..f99b2a6d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" @@ -36,7 +36,7 @@ jobs: python3 setup.py sdist bdist_wheel --python-tag py3 twine check dist/* - name: "Upload Package whl" - uses: "actions/upload-artifact@v3" + uses: "actions/upload-artifact@v4" with: name: "plugin_package" path: "pulp_python/dist/" diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 38a60302..9450eb09 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -28,7 +28,7 @@ jobs: with: fetch-depth: 1 - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 422fa133..17800617 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" @@ -67,10 +67,11 @@ jobs: mkdir -p "pulp_python" working-directory: "." - name: "Download Deprecations" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: "deprecations" + pattern: "deprecations-*" path: "pulp_python" + merge-multiple: true - name: "Print deprecations" run: | cat deprecations-*.txt | sort -u diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index d5f7ebca..b207c880 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -26,7 +26,7 @@ jobs: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 43aee3b1..701d990e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,7 +24,7 @@ jobs: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1cc7dec2..5b9b39ac 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -38,7 +38,7 @@ jobs: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -76,12 +76,12 @@ jobs: fetch-depth: 1 path: "pulp_python" - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: "plugin_package" path: "pulp_python/dist/" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -96,13 +96,13 @@ jobs: echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - name: Download built docs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: "docs.tar" path: "pulp_python" - name: Download Python client docs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: "python-client-docs.tar" path: "pulp_python" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e1715162..0deec163 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,11 +39,11 @@ jobs: repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - - uses: "actions/download-artifact@v3" + - uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_python/dist/" @@ -89,20 +89,20 @@ jobs: .github/workflows/scripts/install_ruby_client.sh shell: "bash" - name: "Upload python client packages" - uses: "actions/upload-artifact@v3" + uses: "actions/upload-artifact@v4" with: name: "python-client.tar" path: "pulp_python/python-python-client.tar" if-no-files-found: "error" - name: "Upload python client docs" - uses: "actions/upload-artifact@v3" + uses: "actions/upload-artifact@v4" with: name: "python-client-docs.tar" path: "pulp_python/python-python-client-docs.tar" if-no-files-found: "error" - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v3" + uses: "actions/upload-artifact@v4" with: name: "ruby-client.tar" path: "pulp_python/python-ruby-client.tar" @@ -115,7 +115,7 @@ jobs: tar -cvf docs/docs.tar docs/_build - name: "Upload built docs" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "docs.tar" path: "pulp_python/docs/docs.tar" @@ -145,12 +145,12 @@ jobs: fetch-depth: 1 path: "pulp_python" - - uses: "actions/download-artifact@v3" + - uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_python/dist/" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -184,7 +184,7 @@ jobs: path: "pulp_python" - name: "Download Python client" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "python-client.tar" path: "pulp_python/" @@ -193,7 +193,7 @@ jobs: run: | tar -xvf python-python-client.tar - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -227,7 +227,7 @@ jobs: path: "pulp_python" - name: "Download Ruby client" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "ruby-client.tar" path: "pulp_python/" @@ -263,7 +263,7 @@ jobs: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -280,13 +280,13 @@ jobs: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - name: "Download built docs" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "docs.tar" path: "pulp_python/" - name: "Download Python client docs" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "python-client-docs.tar" path: "pulp_python/" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff0adfb3..c362a0a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: path: "pulp_python" token: ${{ secrets.RELEASE_TOKEN }} - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e5b52a60..a440b2b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,11 +39,11 @@ jobs: repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - - uses: "actions/download-artifact@v3" + - uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_python/dist/" @@ -119,15 +119,15 @@ jobs: docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt - name: "Upload Deprecations" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: "deprecations" + name: "deprecations-${{ matrix.env.TEST }}" path: "pulp_python/deprecations-${{ matrix.env.TEST }}.txt" if-no-files-found: "error" retention-days: 5 - name: Upload python client packages if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "python-client.tar" path: "pulp_python/python-python-client.tar" @@ -136,7 +136,7 @@ jobs: - name: Upload python client docs if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "python-client-docs.tar" path: "pulp_python/python-python-client-docs.tar" @@ -144,7 +144,7 @@ jobs: retention-days: 5 - name: Upload Ruby client if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "ruby-client.tar" path: "pulp_python/python-ruby-client.tar" @@ -152,7 +152,7 @@ jobs: retention-days: 5 - name: Upload built docs if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "docs.tar" path: "pulp_python/docs/docs.tar" diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index d67136cb..4565da8c 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -19,7 +19,7 @@ jobs: update_backport_labels: runs-on: "ubuntu-latest" steps: - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 87951824..f34476b7 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -29,7 +29,7 @@ jobs: repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" From 48aa1b555bc2f6f1f20fa3af5ce13d5bb59fd76f Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 23 May 2024 15:00:54 +0000 Subject: [PATCH 12/84] Update CI files [noissue] --- .github/template_gitref | 2 +- .github/workflows/build.yml | 1 + .github/workflows/publish.yml | 15 ++++++++++++--- .github/workflows/scripts/publish_client_pypi.sh | 10 ++++------ .github/workflows/test.yml | 7 +++++++ 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.github/template_gitref b/.github/template_gitref index 2232d657..09354118 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-330-g0b7d0c1 +2021.08.26-335-gdf91f79 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f99b2a6d..8ed57631 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,3 +42,4 @@ jobs: path: "pulp_python/dist/" if-no-files-found: "error" retention-days: 5 + overwrite: true diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0deec163..1c225e73 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -88,25 +88,32 @@ jobs: run: | .github/workflows/scripts/install_ruby_client.sh shell: "bash" + - name: "Upload python client packages" uses: "actions/upload-artifact@v4" with: name: "python-client.tar" - path: "pulp_python/python-python-client.tar" + path: | + pulp_python/python-python-client.tar if-no-files-found: "error" + overwrite: true - name: "Upload python client docs" uses: "actions/upload-artifact@v4" with: name: "python-client-docs.tar" - path: "pulp_python/python-python-client-docs.tar" + path: | + pulp_python/python-python-client-docs.tar if-no-files-found: "error" + overwrite: true - name: "Upload ruby client packages" uses: "actions/upload-artifact@v4" with: name: "ruby-client.tar" - path: "pulp_python/python-ruby-client.tar" + path: | + pulp_python/python-ruby-client.tar if-no-files-found: "error" + overwrite: true - name: Build docs run: | export DJANGO_SETTINGS_MODULE=pulpcore.app.settings @@ -119,6 +126,8 @@ jobs: with: name: "docs.tar" path: "pulp_python/docs/docs.tar" + if-no-files-found: "error" + overwrite: true - name: "Logs" if: always() diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index 544c760d..43a31a35 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -24,10 +24,8 @@ RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https:/ if [ "$RESPONSE" == "200" ]; then echo "pulp_python client $VERSION has already been released. Skipping." - exit +else + twine upload -u __token__ -p "$PYPI_API_TOKEN" \ + "dist/pulp_python_client-$VERSION-py3-none-any.whl" \ + "dist/pulp_python-client-$VERSION.tar.gz" fi - -twine upload -u __token__ -p "$PYPI_API_TOKEN" \ -"dist/pulp_python_client-$VERSION-py3-none-any.whl" \ -"dist/pulp_python-client-$VERSION.tar.gz" \ -; diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a440b2b3..4738d1c1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,6 +125,7 @@ jobs: path: "pulp_python/deprecations-${{ matrix.env.TEST }}.txt" if-no-files-found: "error" retention-days: 5 + overwrite: true - name: Upload python client packages if: ${{ env.TEST == 'pulp' }} uses: actions/upload-artifact@v4 @@ -133,6 +134,7 @@ jobs: path: "pulp_python/python-python-client.tar" if-no-files-found: "error" retention-days: 5 + overwrite: true - name: Upload python client docs if: ${{ env.TEST == 'pulp' }} @@ -142,6 +144,7 @@ jobs: path: "pulp_python/python-python-client-docs.tar" if-no-files-found: "error" retention-days: 5 + overwrite: true - name: Upload Ruby client if: ${{ env.TEST == 'pulp' }} uses: actions/upload-artifact@v4 @@ -150,12 +153,16 @@ jobs: path: "pulp_python/python-ruby-client.tar" if-no-files-found: "error" retention-days: 5 + overwrite: true - name: Upload built docs if: ${{ env.TEST == 'docs' }} uses: actions/upload-artifact@v4 with: name: "docs.tar" path: "pulp_python/docs/docs.tar" + if-no-files-found: "error" + retention-days: 5 + overwrite: true - name: "Logs" if: always() From 19e069e88cc67222cf0cfee7e8f4ee9d8fe82694 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 30 May 2024 20:51:52 +0000 Subject: [PATCH 13/84] Update CI files [noissue] --- .ci/scripts/collect_changes.py | 8 +- .github/template_gitref | 2 +- CHANGES.md | 437 +++++++++++++++++++++++ CHANGES.rst | 617 --------------------------------- CHANGES/.TEMPLATE.md | 39 +++ MANIFEST.in | 2 +- docs/changes.rst | 7 +- pyproject.toml | 10 +- 8 files changed, 492 insertions(+), 630 deletions(-) create mode 100644 CHANGES.md delete mode 100644 CHANGES.rst create mode 100644 CHANGES/.TEMPLATE.md diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 49153880..b6c64eb4 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -1,3 +1,4 @@ +#!/bin/env python3 # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use @@ -8,13 +9,14 @@ import itertools import os import re +import tomllib -import toml from git import GitCommandError, Repo from packaging.version import parse as parse_version # Read Towncrier settings -tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"] +with open("pyproject.toml", "rb") as fp: + tc_settings = tomllib.load(fp)["tool"]["towncrier"] CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( @@ -79,7 +81,7 @@ def main(): old_length = len(main_changes) for branch in branches: - print(f"Looking at branch {branch}") + print(f"Looking for './{CHANGELOG_FILE}' at branch {branch}") try: changelog = get_changelog(repo, branch) except GitCommandError: diff --git a/.github/template_gitref b/.github/template_gitref index 09354118..bfe538dd 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-335-gdf91f79 +2021.08.26-337-g7c7a09a diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..84412881 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,437 @@ +# Changelog + +[//]: # (You should *NOT* be adding new change log entries to this file, this) +[//]: # (file is managed by towncrier. You *may* edit previous change logs to) +[//]: # (fix problems like typo corrections or such.) +[//]: # (To add a new change log entry, please see the contributing docs.) +[//]: # (WARNING: Don't drop the towncrier directive!) + +[//]: # (towncrier release notes start) + +## 3.11.1 (2024-04-11) {: #3.11.1 } + +### Bugfixes + +- Fixed tls_validation not being disabled when set to false on the remote. + [#653](https://github.com/pulp/pulp_python/issues/653) + +--- + +## 3.11.0 (2023-11-08) {: #3.11.0 } + +### Features + +- Added pulpcore 3.40 compatibility. +- Added import export support of python content. + [#579](https://github.com/pulp/pulp_python/issues/579) + +--- + +## 3.10.0 (2023-05-17) {: #3.10.0 } + +### Features + +- Added compatibility for pulpcore 3.25, pulpcore support is now >=3.25,<3.40. + [#605](https://github.com/pulp/pulp_python/issues/605) + +--- + +## 3.9.0 (2023-03-17) {: #3.9.0 } + +### Features + +- Added version filter to package list endpoint. + [#577](https://github.com/pulp/pulp_python/issues/577) +- Allow duplicate uploads to return existing packages instead of erring. + [#590](https://github.com/pulp/pulp_python/issues/590) + +### Bugfixes + +- Fixed pull-through caching ignoring remote proxy settings. + [#553](https://github.com/pulp/pulp_python/issues/553) +- Changed includes and excludes openapi schema to report as array of strings instead of object. + [#576](https://github.com/pulp/pulp_python/issues/576) +- Fixed syncing ignoring remote proxy. + [#581](https://github.com/pulp/pulp_python/issues/581) +- Fixed duplicate operationID for generated PyPI simple endpoints schema. + [#594](https://github.com/pulp/pulp_python/issues/594) + +--- + +## 3.8.0 (2022-12-19) {: #3.8.0 } + +### Bugfixes + +- Fixed syncing failing when using bandersnatch 5.3.0 + [#554](https://github.com/pulp/pulp_python/issues/554) +- Prevent .netrc file from being read on syncs. + [#566](https://github.com/pulp/pulp_python/issues/566) +- Fix 500 error when pip installing using object storage. + [#572](https://github.com/pulp/pulp_python/issues/572) + +### Improved Documentation + +- Documented `pulp_python` specific settings. + [#571](https://github.com/pulp/pulp_python/issues/571) + +--- + +## 3.7.3 (2022-10-06) {: #3.7.3 } + +### Bugfixes + +- Prevent .netrc file from being read on syncs. + [#566](https://github.com/pulp/pulp_python/issues/566) + +--- + +## 3.7.2 (2022-08-04) {: #3.7.2 } + +### Bugfixes + +- Fixed syncing failing when using bandersnatch 5.3.0 + [#554](https://github.com/pulp/pulp_python/issues/554) + +--- + +## 3.7.1 (2022-06-29) {: #3.7.1 } + +No significant changes. + +--- + +## 3.7.0 (2022-06-22) {: #3.7.0 } + +### Features + +- Added ability to fully sync repositories that don't support the PyPI XMLRPC endpoints. Full Pulp-to-Pulp syncing is now available. + [#462](https://github.com/pulp/pulp_python/issues/462) + +### Bugfixes + +- Ensured temporary package uploads are written to worker's directory instead of /tmp. + [#505](https://github.com/pulp/pulp_python/issues/505) + +### Misc + +- [#503](https://github.com/pulp/pulp_python/issues/503) + +--- + +## 3.6.1 (2022-08-19) {: #3.6.1 } + +### Bugfixes + +- Fixed syncing failing when using bandersnatch 5.3.0 + [#554](https://github.com/pulp/pulp_python/issues/554) + +--- + +## 3.6.0 (2021-12-15) {: #3.6.0 } + +### Features + +- `pulp_python` now supports pull-through caching. Add a remote to a distribution to enable this feature. + [#381](https://github.com/pulp/pulp_python/issues/381) +- Enable Azure support + [#458](https://github.com/pulp/pulp_python/issues/458) + +### Bugfixes + +- Fixed proxy url not being passed during sync + [#433](https://github.com/pulp/pulp_python/issues/433) +- Changed the use of `dispatch` to match the signature from pulpcore>=3.15. + [#443](https://github.com/pulp/pulp_python/issues/443) +- Fixed package name normalization issue preventing installing packages with "." or "_" in their names. + [#467](https://github.com/pulp/pulp_python/issues/467) + +--- + +## 3.5.2 (2021-10-05) {: #3.5.2 } + +### Bugfixes + +- Fixed proxy url not being passed during sync + (backported from #445) + [#436](https://github.com/pulp/pulp_python/issues/436) +- Changed the use of `dispatch` to match the signature from pulpcore>=3.15. + (backported from #443) + [#446](https://github.com/pulp/pulp_python/issues/446) + +--- + +## 3.5.1 (2021-09-10) {: #3.5.1 } + +### Bugfixes + +- Fixed proxy url not being passed during sync + (backported from #433) + [#436](https://github.com/pulp/pulp_python/issues/436) + +--- + +## 3.5.0 (2021-08-30) {: #3.5.0 } + +### Features + +- Python package content can now be filtered by their sha256 + [#404](https://github.com/pulp/pulp_python/issues/404) +- Added new setting `PYPI_API_HOSTNAME` that is used to form a distribution's `base_url`. Defaults to the machine's FQDN. + [#412](https://github.com/pulp/pulp_python/issues/412) +- Enabled reclaim disk feature provided by pulpcore 3.15+. + [#425](https://github.com/pulp/pulp_python/issues/425) + +### Bugfixes + +- Fixed twine upload failing when using remote storage backends + [#400](https://github.com/pulp/pulp_python/issues/400) +- Fixed improper metadata serving when using publications with S3 storage + [#413](https://github.com/pulp/pulp_python/issues/413) + +### Deprecations and Removals + +- Dropped support for Python < 3.8. + [#402](https://github.com/pulp/pulp_python/issues/402) + +### Misc + +- [#408](https://github.com/pulp/pulp_python/issues/408), [#427](https://github.com/pulp/pulp_python/issues/427) + +--- + +## 3.4.1 (2021-08-24) {: #3.4.1 } + +### Features + +- Python package content can now be filtered by their sha256 + (backported from #404) + [#419](https://github.com/pulp/pulp_python/issues/419) + +### Bugfixes + +- Fixed improper metadata serving when using publications with S3 storage + (backported from #413) + [#418](https://github.com/pulp/pulp_python/issues/418) +- Fixed twine upload failing when using remote storage backends + (backported from #400) + [#420](https://github.com/pulp/pulp_python/issues/420) + +--- + +3.4.0 (2021-06-17) + +### Features + +- Added `twine` (and other similar Python tools) package upload support + [#342](https://github.com/pulp/pulp_python/issues/342) +- PyPI endpoints are now available at `/pypi/{base_path}/` + [#376](https://github.com/pulp/pulp_python/issues/376) +- Changed the global uniqueness constraint for `PythonPackageContent` to its sha256 digest + [#380](https://github.com/pulp/pulp_python/issues/380) + +### Bugfixes + +- Added missing fields to PyPI live JSON API to be compliant with core metadata version 2.1 + [#352](https://github.com/pulp/pulp_python/issues/352) +- Fixed sync to use default concurrency (10) when download_concurrency was not specified + [#391](https://github.com/pulp/pulp_python/issues/391) + +--- + +## 3.3.0 (2021-05-27) {: #3.3.0 } + +### Features + +- Add support for automatic publishing and distributing. + [#365](https://github.com/pulp/pulp_python/issues/365) + +### Bugfixes + +- Fixed publications publishing more content than was in the repository + [#362](https://github.com/pulp/pulp_python/issues/362) + +### Improved Documentation + +- Update syntax in doc for cli repository content add command + [#368](https://github.com/pulp/pulp_python/issues/368) + +### Misc + +- [#347](https://github.com/pulp/pulp_python/issues/347), [#360](https://github.com/pulp/pulp_python/issues/360), [#371](https://github.com/pulp/pulp_python/issues/371) + +--- + +## 3.2.0 (2021-04-14) {: #3.2.0 } + +### Features + +- Added new sync filter keep_latest_packages to specify how many latest versions of packages to sync + [#339](https://github.com/pulp/pulp_python/issues/339) +- Added new sync filters package_types and exclude_platforms to specify package types to sync + [#341](https://github.com/pulp/pulp_python/issues/341) + +### Misc + +- [#354](https://github.com/pulp/pulp_python/issues/354) + +--- + +## 3.1.0 (2021-03-12) {: #3.1.0 } + +### Features + +- Python content can now be filtered by requires_python + [#3629](https://pulp.plan.io/issues/3629) + +### Improved Documentation + +- Updated workflows to use Pulp CLI commands + [#8364](https://pulp.plan.io/issues/8364) + +--- + +## 3.0.0 (2021-01-12) {: #3.0.0 } + +### Bugfixes + +- Remote proxy settings are now passed to Bandersnatch while syncing + [#7864](https://pulp.plan.io/issues/7864) + +### Improved Documentation + +- Added bullet list of Python Plugin features and a tech preview page for new experimental features + [#7628](https://pulp.plan.io/issues/7628) + +--- + +## 3.0.0b12 (2020-11-05) + +### Features + +- Pulp Python can now fully mirror all packages from PyPi + [#985](https://pulp.plan.io/issues/985) +- Implemented PyPi's json API at content endpoint '/pypi/{package-name}/json'. Pulp can now perform basic syncing on other Pulp Python instances. + [#2886](https://pulp.plan.io/issues/2886) +- Pulp Python now uses Bandersnatch to perform syncing and filtering of package metadata + [#6930](https://pulp.plan.io/issues/6930) + +### Bugfixes + +- Sync now includes python package's classifiers in the content unit + [#3627](https://pulp.plan.io/issues/3627) +- Policy can now be specified when creating a remote from a Bandersnatch config + [#7331](https://pulp.plan.io/issues/7331) +- Includes/excludes/prereleases fields are now properly set in a remote from Bandersnatch config + [#7392](https://pulp.plan.io/issues/7392) + +### Improved Documentation + +- Fixed makemigrations commands in the install docs + [#5386](https://pulp.plan.io/issues/5386) + +### Misc + +- [#6875](https://pulp.plan.io/issues/6875), [#7401](https://pulp.plan.io/issues/7401) + +--- + +## 3.0.0b11 (2020-08-18) + +Compatibility update for pulpcore 3.6 + +--- + +## 3.0.0b10 (2020-08-05) + +### Features + +- Added a new endpoint to remotes "/from_bandersnatch" that allows for Python remote creation from a Bandersnatch config file. + [#6929](https://pulp.plan.io/issues/6929) + +### Bugfixes + +- Including requirements.txt on MANIFEST.in + [#6891](https://pulp.plan.io/issues/6891) +- Updating API to not return publications that aren't complete. + [#6987](https://pulp.plan.io/issues/6987) +- Fixed an issue that prevented 'on_demand' content from being published. + [#7128](https://pulp.plan.io/issues/7128) + +### Improved Documentation + +- Change the commands for publication and distribution on the publish workflow to use their respective scripts already defined in _scripts. + [#6877](https://pulp.plan.io/issues/6877) +- Updated sync.sh, publication.sh and distribution.sh in docs/_scripts to reference wait_until_task_finished function from base.sh + [#6918](https://pulp.plan.io/issues/6918) + +--- + +## 3.0.0b9 (2020-06-01) + +### Features + +- Add upload functionality to the python contents endpoints. + [#5464](https://pulp.plan.io/issues/5464) + +### Bugfixes + +- Fixed the 500 error returned by the OpenAPI schema endpoint. + [#5452](https://pulp.plan.io/issues/5452) + +### Improved Documentation + +- Change the prefix of Pulp services from pulp-* to pulpcore-* + [#4554](https://pulp.plan.io/issues/4554) +- Added "python/python/" to fix two commands in repo.sh, fixed export command in sync.sh + [#6790](https://pulp.plan.io/issues/6790) +- Added "index.html" to the relative_path field for both project_metadata and index_metadata. Added a "/" to fix the link in the simple_index_template. + [#6792](https://pulp.plan.io/issues/6792) +- Updated the workflow documentation for upload.html. Fixed the workflow commands and added more details to the instructions. + [#6854](https://pulp.plan.io/issues/6854) + +### Deprecations and Removals + +- Change _id, _created, _last_updated, _href to pulp_id, pulp_created, pulp_last_updated, pulp_href + [#5457](https://pulp.plan.io/issues/5457) + +- Remove "_" from _versions_href, _latest_version_href + [#5548](https://pulp.plan.io/issues/5548) + +- Removing base field: _type . + [#5550](https://pulp.plan.io/issues/5550) + +- Sync is no longer available at the {remote_href}/sync/ repository={repo_href} endpoint. Instead, use POST {repo_href}/sync/ remote={remote_href}. + + Creating / listing / editing / deleting python repositories is now performed on /pulp/api/v3/python/python/ instead of /pulp/api/v3/repositories/. Only python content can be present in a python repository, and only a python repository can hold python content. + [#5625](https://pulp.plan.io/issues/5625) + +### Misc + +- [#remotetests](https://pulp.plan.io/issues/remotetests), [#4681](https://pulp.plan.io/issues/4681), [#4682](https://pulp.plan.io/issues/4682), [#5304](https://pulp.plan.io/issues/5304), [#5471](https://pulp.plan.io/issues/5471), [#5580](https://pulp.plan.io/issues/5580), [#5701](https://pulp.plan.io/issues/5701) + +--- + +## 3.0.0b8 (2019-09-16) + +### Misc + +- [#4681](https://pulp.plan.io/issues/4681) + +--- + +## 3.0.0b7 (2019-08-01) + +### Features + +- Users can upload a file to create content and optionally add to a repo in one step known as + one-shot upload + [#4396](https://pulp.plan.io/issues/4396) +- Override the Remote's serializer to allow policy='on_demand' and policy='streamed'. + [#4990](https://pulp.plan.io/issues/4990) + +### Improved Documentation + +- Switch to using [towncrier](https://github.com/hawkowl/towncrier) for better release notes. + [#4875](https://pulp.plan.io/issues/4875) + +--- diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index 1e38e617..00000000 --- a/CHANGES.rst +++ /dev/null @@ -1,617 +0,0 @@ -========= -Changelog -========= - -.. - You should *NOT* be adding new change log entries to this file, this - file is managed by towncrier. You *may* edit previous change logs to - fix problems like typo corrections or such. - To add a new change log entry, please see - https://docs.pulpproject.org/en/3.0/nightly/contributing/git.html#changelog-update - - WARNING: Don't drop the next directive! - -.. towncrier release notes start - -3.11.1 (2024-04-11) -=================== - -Bugfixes --------- - -- Fixed tls_validation not being disabled when set to false on the remote. - `#653 `__ - - ----- - - -3.11.0 (2023-11-08) -=================== - - -Features --------- - -- Added pulpcore 3.40 compatibility. -- Added import export support of python content. - `#579 `__ - - ----- - - -3.10.0 (2023-05-17) -=================== - - -Features --------- - -- Added compatibility for pulpcore 3.25, pulpcore support is now >=3.25,<3.40. - `#605 `__ - - ----- - - -3.9.0 (2023-03-17) -================== - - -Features --------- - -- Added version filter to package list endpoint. - `#577 `__ -- Allow duplicate uploads to return existing packages instead of erring. - `#590 `__ - - -Bugfixes --------- - -- Fixed pull-through caching ignoring remote proxy settings. - `#553 `__ -- Changed includes and excludes openapi schema to report as array of strings instead of object. - `#576 `__ -- Fixed syncing ignoring remote proxy. - `#581 `__ -- Fixed duplicate operationID for generated PyPI simple endpoints schema. - `#594 `__ - - ----- - - -3.8.0 (2022-12-19) -================== - - -Bugfixes --------- - -- Fixed syncing failing when using bandersnatch 5.3.0 - `#554 `__ -- Prevent .netrc file from being read on syncs. - `#566 `__ -- Fix 500 error when pip installing using object storage. - `#572 `__ - - -Improved Documentation ----------------------- - -- Documented ``pulp_python`` specific settings. - `#571 `__ - - ----- - - -3.7.3 (2022-10-06) -================== - - -Bugfixes --------- - -- Prevent .netrc file from being read on syncs. - `#566 `__ - - ----- - - -3.7.2 (2022-08-04) -================== - - -Bugfixes --------- - -- Fixed syncing failing when using bandersnatch 5.3.0 - `#554 `__ - - ----- - - -3.7.1 (2022-06-29) -================== - - -No significant changes. - - ----- - - -3.7.0 (2022-06-22) -================== - - -Features --------- - -- Added ability to fully sync repositories that don't support the PyPI XMLRPC endpoints. Full Pulp-to-Pulp syncing is now available. - `#462 `__ - - -Bugfixes --------- - -- Ensured temporary package uploads are written to worker's directory instead of /tmp. - `#505 `__ - - -Misc ----- - -- `#503 `__ - - ----- - - -3.6.1 (2022-08-19) -================== - - -Bugfixes --------- - -- Fixed syncing failing when using bandersnatch 5.3.0 - `#554 `__ - - ----- - - -3.6.0 (2021-12-15) -================== - - -Features --------- - -- ``pulp_python`` now supports pull-through caching. Add a remote to a distribution to enable this feature. - `#381 `_ -- Enable Azure support - `#458 `_ - - -Bugfixes --------- - -- Fixed proxy url not being passed during sync - `#433 `_ -- Changed the use of ``dispatch`` to match the signature from pulpcore>=3.15. - `#443 `_ -- Fixed package name normalization issue preventing installing packages with "." or "_" in their names. - `#467 `_ - - ----- - - -3.5.2 (2021-10-05) -================== - - -Bugfixes --------- - -- Fixed proxy url not being passed during sync - (backported from #445) - `#436 `_ -- Changed the use of ``dispatch`` to match the signature from pulpcore>=3.15. - (backported from #443) - `#446 `_ - - ----- - - -3.5.1 (2021-09-10) -================== - - -Bugfixes --------- - -- Fixed proxy url not being passed during sync - (backported from #433) - `#436 `_ - - ----- - - -3.5.0 (2021-08-30) -================== - - -Features --------- - -- Python package content can now be filtered by their sha256 - `#404 `_ -- Added new setting ``PYPI_API_HOSTNAME`` that is used to form a distribution's ``base_url``. Defaults to the machine's FQDN. - `#412 `_ -- Enabled reclaim disk feature provided by pulpcore 3.15+. - `#425 `_ - - -Bugfixes --------- - -- Fixed twine upload failing when using remote storage backends - `#400 `_ -- Fixed improper metadata serving when using publications with S3 storage - `#413 `_ - - -Deprecations and Removals -------------------------- - -- Dropped support for Python < 3.8. - `#402 `_ - - -Misc ----- - -- `#408 `_, `#427 `_ - - ----- - - -3.4.1 (2021-08-24) -================== - - -Features --------- - -- Python package content can now be filtered by their sha256 - (backported from #404) - `#419 `_ - - -Bugfixes --------- - -- Fixed improper metadata serving when using publications with S3 storage - (backported from #413) - `#418 `_ -- Fixed twine upload failing when using remote storage backends - (backported from #400) - `#420 `_ - - ----- - - -3.4.0 (2021-06-17) - -Features --------- - -- Added ``twine`` (and other similar Python tools) package upload support - `#342 `_ -- PyPI endpoints are now available at ``/pypi/{base_path}/`` - `#376 `_ -- Changed the global uniqueness constraint for ``PythonPackageContent`` to its sha256 digest - `#380 `_ - - -Bugfixes --------- - -- Added missing fields to PyPI live JSON API to be compliant with core metadata version 2.1 - `#352 `_ -- Fixed sync to use default concurrency (10) when download_concurrency was not specified - `#391 `_ - - ----- - - -3.3.0 (2021-05-27) -================== - - -Features --------- - -- Add support for automatic publishing and distributing. - `#365 `_ - - -Bugfixes --------- - -- Fixed publications publishing more content than was in the repository - `#362 `_ - - -Improved Documentation ----------------------- - -- Update syntax in doc for cli repository content add command - `#368 `_ - - -Misc ----- - -- `#347 `_, `#360 `_, `#371 `_ - - ----- - - -3.2.0 (2021-04-14) -================== - - -Features --------- - -- Added new sync filter `keep_latest_packages` to specify how many latest versions of packages to sync - `#339 `_ -- Added new sync filters `package_types` and `exclude_platforms` to specify package types to sync - `#341 `_ - - -Misc ----- - -- `#354 `_ - - ----- - - -3.1.0 (2021-03-12) -================== - - -Features --------- - -- Python content can now be filtered by requires_python - `#3629 `_ - - -Improved Documentation ----------------------- - -- Updated workflows to use Pulp CLI commands - `#8364 `_ - - ----- - - -3.0.0 (2021-01-12) -================== - - -Bugfixes --------- - -- Remote proxy settings are now passed to Bandersnatch while syncing - `#7864 `_ - - -Improved Documentation ----------------------- - -- Added bullet list of Python Plugin features and a tech preview page for new experimental features - `#7628 `_ - - ----- - - -3.0.0b12 (2020-11-05) -===================== - - -Features --------- - -- Pulp Python can now fully mirror all packages from PyPi - `#985 `_ -- Implemented PyPi's json API at content endpoint '/pypi/{package-name}/json'. Pulp can now perform basic syncing on other Pulp Python instances. - `#2886 `_ -- Pulp Python now uses Bandersnatch to perform syncing and filtering of package metadata - `#6930 `_ - - -Bugfixes --------- - -- Sync now includes python package's classifiers in the content unit - `#3627 `_ -- Policy can now be specified when creating a remote from a Bandersnatch config - `#7331 `_ -- Includes/excludes/prereleases fields are now properly set in a remote from Bandersnatch config - `#7392 `_ - - -Improved Documentation ----------------------- - -- Fixed makemigrations commands in the install docs - `#5386 `_ - - -Misc ----- - -- `#6875 `_, `#7401 `_ - - ----- - - -3.0.0b11 (2020-08-18) -===================== - - -Compatibility update for pulpcore 3.6 - - ----- - - -3.0.0b10 (2020-08-05) -===================== - - -Features --------- - -- Added a new endpoint to remotes "/from_bandersnatch" that allows for Python remote creation from a Bandersnatch config file. - `#6929 `_ - - -Bugfixes --------- - -- Including requirements.txt on MANIFEST.in - `#6891 `_ -- Updating API to not return publications that aren't complete. - `#6987 `_ -- Fixed an issue that prevented 'on_demand' content from being published. - `#7128 `_ - - -Improved Documentation ----------------------- - -- Change the commands for publication and distribution on the publish workflow to use their respective scripts already defined in _scripts. - `#6877 `_ -- Updated sync.sh, publication.sh and distribution.sh in docs/_scripts to reference wait_until_task_finished function from base.sh - `#6918 `_ - - ----- - - -3.0.0b9 (2020-06-01) -==================== - - -Features --------- - -- Add upload functionality to the python contents endpoints. - `#5464 `_ - - -Bugfixes --------- - -- Fixed the 500 error returned by the OpenAPI schema endpoint. - `#5452 `_ - - -Improved Documentation ----------------------- - -- Change the prefix of Pulp services from pulp-* to pulpcore-* - `#4554 `_ -- Added "python/python/" to fix two commands in repo.sh, fixed export command in sync.sh - `#6790 `_ -- Added "index.html" to the relative_path field for both project_metadata and index_metadata. Added a "/" to fix the link in the simple_index_template. - `#6792 `_ -- Updated the workflow documentation for upload.html. Fixed the workflow commands and added more details to the instructions. - `#6854 `_ - - -Deprecations and Removals -------------------------- - -- Change `_id`, `_created`, `_last_updated`, `_href` to `pulp_id`, `pulp_created`, `pulp_last_updated`, `pulp_href` - `#5457 `_ -- Remove "_" from `_versions_href`, `_latest_version_href` - `#5548 `_ -- Removing base field: `_type` . - `#5550 `_ -- Sync is no longer available at the {remote_href}/sync/ repository={repo_href} endpoint. Instead, use POST {repo_href}/sync/ remote={remote_href}. - - Creating / listing / editing / deleting python repositories is now performed on /pulp/api/v3/python/python/ instead of /pulp/api/v3/repositories/. Only python content can be present in a python repository, and only a python repository can hold python content. - `#5625 `_ - - -Misc ----- - -- `#remotetests `_, `#4681 `_, `#4682 `_, `#5304 `_, `#5471 `_, `#5580 `_, `#5701 `_ - - ----- - - -3.0.0b8 (2019-09-16) -==================== - - -Misc ----- - -- `#4681 `_ - - ----- - - -3.0.0b7 (2019-08-01) -==================== - - -Features --------- - -- Users can upload a file to create content and optionally add to a repo in one step known as - one-shot upload - `#4396 `_ -- Override the Remote's serializer to allow policy='on_demand' and policy='streamed'. - `#4990 `_ - - -Improved Documentation ----------------------- - -- Switch to using `towncrier `_ for better release notes. - `#4875 `_ - - ----- - - diff --git a/CHANGES/.TEMPLATE.md b/CHANGES/.TEMPLATE.md new file mode 100644 index 00000000..2308193b --- /dev/null +++ b/CHANGES/.TEMPLATE.md @@ -0,0 +1,39 @@ +{# TOWNCRIER TEMPLATE #} +{% for section, _ in sections.items() %} +{%- set section_slug = "-" + section|replace(" ", "-")|replace("_", "-")|lower %} +{%- if section %} + +### {{section}} {: #{{versiondata.version}}{{section_slug}} } +{% else %} +{%- set section_slug = "" %} +{% endif %} +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section]%} + +#### {{ definitions[category]['name'] }} {: #{{versiondata.version}}{{section_slug}}-{{category}} } + +{% if definitions[category]['showcontent'] %} +{% for text, values in sections[section][category].items() %} +- {{ text }} +{% if values %} + {{ values|join(',\n ') }} +{% endif %} +{% endfor %} +{% else %} +- {{ sections[section][category]['']|join(', ') }} +{% endif %} +{% if sections[section][category]|length == 0 %} + +No significant changes. +{% else %} +{% endif %} +{% endfor %} +{% else %} + +No significant changes. +{% endif %} +{% endfor %} + +--- + + diff --git a/MANIFEST.in b/MANIFEST.in index aca0d594..f9759e6f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ include LICENSE include requirements.txt include pyproject.toml -include CHANGES.rst +include CHANGES.md include COMMITMENT include COPYRIGHT include functest_requirements.txt diff --git a/docs/changes.rst b/docs/changes.rst index 3e96678c..96e8eb90 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,5 +1,4 @@ -.. _pulp_python-changes: +Changes +********* -.. include:: ../CHANGES.rst - -.. include:: ../HISTORY.rst +Removed due to docs migration process. diff --git a/pyproject.toml b/pyproject.toml index cba5c3f3..6f2c221a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,12 @@ [tool.towncrier] package = "pulp_python" -filename = "CHANGES.rst" +filename = "CHANGES.md" directory = "CHANGES/" -title_format = "{version} ({project_date})" -template = "CHANGES/.TEMPLATE.rst" -issue_format = "`#{issue} `__" +title_format = "## {version} ({project_date}) {{: #{version} }}" +template = "CHANGES/.TEMPLATE.md" +issue_format = "[#{issue}](https://github.com/pulp/pulp_python/issues/{issue})" +start_string = "[//]: # (towncrier release notes start)\n" +underlines = ["", "", ""] [tool.check-manifest] ignore = [ From 9efd1926801228bd9ff72e1c6f4fcae64a20dc1a Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 12 Jun 2024 13:09:27 +0000 Subject: [PATCH 14/84] Update CI files [noissue] --- .ci/ansible/settings.py.j2 | 2 +- .github/template_gitref | 2 +- .github/workflows/scripts/before_script.sh | 6 ----- .github/workflows/scripts/install.sh | 26 +++------------------- 4 files changed, 5 insertions(+), 31 deletions(-) diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 024df4e4..4ed28f42 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -49,7 +49,7 @@ AZURE_CONTAINER = "pulp-test" AZURE_LOCATION = "pulp3" AZURE_OVERWRITE_FILES = True AZURE_URL_EXPIRATION_SECS = 120 -AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;' +AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' {% endif %} {% if gcp_test | default(false) %} diff --git a/.github/template_gitref b/.github/template_gitref index bfe538dd..9122ea22 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-337-g7c7a09a +2021.08.26-339-gf0d923e diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 175db1fe..5c0072c4 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -36,12 +36,6 @@ tail -v -n +1 .ci/ansible/Containerfile cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" cmd_prefix bash -c "usermod -a -G wheel pulp" -SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds") -if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then - # Many functional tests require these - cmd_prefix dnf install -yq lsof which -fi - if [[ "${REDIS_DISABLED:-false}" == true ]]; then cmd_prefix bash -c "s6-rc -d change redis" echo "The Redis service was disabled for $TEST" diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 6c9fdd4a..356eba1e 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -104,18 +104,12 @@ pulp_scenario_env: {}\ fi if [ "$TEST" = "azure" ]; then - mkdir -p azurite - cd azurite - openssl req -newkey rsa:2048 -x509 -nodes -keyout azkey.pem -new -out azcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-azurite" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN" - sudo cp azcert.pem /usr/local/share/ca-certificates/azcert.crt - sudo dpkg-reconfigure ca-certificates - cd .. sed -i -e '/^services:/a \ - name: ci-azurite\ image: mcr.microsoft.com/azure-storage/azurite\ volumes:\ - ./azurite:/etc/pulp\ - command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml + command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ pulp_scenario_settings: null\ pulp_scenario_env: {}\ @@ -151,31 +145,17 @@ sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certi # Hack: adding pulp CA to certifi.where() CERTIFI=$(python -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERTIFI" > /dev/null -if [[ "$TEST" = "azure" ]]; then - cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a "$CERTIFI" > /dev/null -fi # Hack: adding pulp CA to default CA file CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)') -cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null +cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERT" > /dev/null # Updating certs sudo update-ca-certificates echo ::endgroup:: -# Add our azcert.crt certificate to the container image along with the certificates from certifi -# so that we can use HTTPS with our fake Azure CI. certifi is self-contained and doesn't allow -# extension or modification of the trust store, so we do a weird and hacky thing (above) where we just -# overwrite or append to certifi's trust store behind it's back. -# -# We do this for both the CI host and the CI image. if [[ "$TEST" = "azure" ]]; then - AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())') - PULPCERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') - cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a "$PULPCERTIFI" > /dev/null - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /etc/pki/tls/cert.pem > /dev/null - AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://ci-azurite:10000/devstoreaccount1;' + AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING fi From 6b37c3cdebdfbd407726eae9699a781d2382f818 Mon Sep 17 00:00:00 2001 From: Gerrod Date: Tue, 25 Jun 2024 13:59:19 -0400 Subject: [PATCH 15/84] Merge pull request #684 from gerrod3/ft-remove-bindings Remove bindings from functest requirements (cherry picked from commit 8695c9bac814440cc6f66d241271060a992fd78f) --- functest_requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/functest_requirements.txt b/functest_requirements.txt index 17cfad8f..0beaa4d5 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -3,5 +3,3 @@ pytest<8 lxml twine pypi-simple -pulpcore-client -pulp-python-client From 580c6768e220ec10c709d5a474003c98be2f55a9 Mon Sep 17 00:00:00 2001 From: Gerrod Date: Thu, 27 Jun 2024 10:03:41 -0400 Subject: [PATCH 16/84] Merge pull request #692 from gerrod3/package-type-filter-fix Fix package_types filter breaking others (cherry picked from commit a37d040b901d70a527c208fab9edef486039a205) --- CHANGES/691.bugfix | 1 + pulp_python/app/tasks/sync.py | 2 +- pulp_python/tests/functional/api/test_sync.py | 17 ++++ pulp_python/tests/functional/conftest.py | 92 +++++++++++++++++++ pulp_python/tests/functional/constants.py | 1 + 5 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 CHANGES/691.bugfix diff --git a/CHANGES/691.bugfix b/CHANGES/691.bugfix new file mode 100644 index 00000000..448af729 --- /dev/null +++ b/CHANGES/691.bugfix @@ -0,0 +1 @@ +Fixed the `package_types` filter breaking other remote filters. diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 3af26a59..1293c613 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -83,7 +83,7 @@ def create_bandersnatch_config(remote): config["plugins"]["enabled"] += "prerelease_release\n" if remote.package_types: rrfm = "regex_release_file_metadata" - config["plugins"]["enabled"] += rrfm + config["plugins"]["enabled"] += f"{rrfm}\n" if not config.has_section(rrfm): config.add_section(rrfm) config[rrfm]["any:release_file.packagetype"] = "\n".join(remote.package_types) diff --git a/pulp_python/tests/functional/api/test_sync.py b/pulp_python/tests/functional/api/test_sync.py index b64e8404..79a07b0c 100644 --- a/pulp_python/tests/functional/api/test_sync.py +++ b/pulp_python/tests/functional/api/test_sync.py @@ -641,6 +641,23 @@ def test_no_platform_sync(self): ) +@pytest.mark.parallel +def test_sync_multiple_filters( + python_repo_with_sync, python_remote_factory, python_content_summary +): + """Tests sync with multiple filters.""" + remote = python_remote_factory( + includes=PYTHON_LG_PROJECT_SPECIFIER, + package_types=["bdist_wheel"], + keep_latest_packages=1, + prereleases=False + ) + repo = python_repo_with_sync(remote) + + summary = python_content_summary(repository_version=repo.latest_version_href) + assert summary.present["python.python"]["count"] == PYTHON_LG_FIXTURE_COUNTS["multi"] + + @pytest.mark.parallel def test_proxy_sync( python_repo, diff --git a/pulp_python/tests/functional/conftest.py b/pulp_python/tests/functional/conftest.py index 01a0552d..732695e4 100644 --- a/pulp_python/tests/functional/conftest.py +++ b/pulp_python/tests/functional/conftest.py @@ -1,8 +1,10 @@ import pytest +import subprocess import uuid from pulp_smash.pulp3.utils import gen_distribution from pulp_python.tests.functional.utils import gen_python_remote +from pulp_python.tests.functional.constants import PYTHON_URL, PYTHON_EGG_FILENAME from pulpcore.client.pulp_python import ( ApiClient, @@ -110,3 +112,93 @@ def _gen_python_remote(**kwargs): return gen_object_with_cleanup(python_remote_api_client, body) yield _gen_python_remote + + +@pytest.fixture +def python_repo_with_sync( + python_repo_api_client, python_repo_factory, python_remote_factory, monitor_task +): + """A factory to generate a Python Repository synced with the passed in Remote.""" + def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): + kwargs = {} + if pulp_domain := body.get("pulp_domain"): + kwargs["pulp_domain"] = pulp_domain + remote = remote or python_remote_factory(**kwargs) + repo = repository or python_repo_factory(**body) + sync_body = {"mirror": mirror, "remote": remote.pulp_href} + monitor_task(python_repo_api_client.sync(repo.pulp_href, sync_body).task) + return python_repo_api_client.read(repo.pulp_href) + + yield _gen_python_repo_sync + + +@pytest.fixture +def download_python_file(tmp_path, http_get): + """Download a Python file and return its path.""" + def _download_python_file(relative_path, url): + file_path = tmp_path / relative_path + with open(file_path, mode="wb") as f: + f.write(http_get(url)) + return file_path + + yield _download_python_file + + +@pytest.fixture +def python_file(download_python_file): + """Get a default (shelf-reader.tar.gz) Python file.""" + return download_python_file(PYTHON_EGG_FILENAME, PYTHON_URL) + + +@pytest.fixture +def python_content_factory(python_content_api_client, download_python_file, monitor_task): + """A factory to create a Python Package Content.""" + def _gen_python_content(relative_path=PYTHON_EGG_FILENAME, url=None, **body): + body["relative_path"] = relative_path + if url: + body["file"] = download_python_file(relative_path, url) + elif not any(x in body for x in ("artifact", "file", "upload")): + body["file"] = download_python_file(PYTHON_EGG_FILENAME, PYTHON_URL) + if repo := body.get("repository"): + repo_href = repo if isinstance(repo, str) else repo.pulp_href + body["repository"] = repo_href + + task = python_content_api_client.create(**body).task + response = monitor_task(task) + return python_content_api_client.read(response.created_resources[0]) + + yield _gen_python_content + + +# Utility fixtures + + +@pytest.fixture +def shelf_reader_cleanup(): + """Take care of uninstalling shelf-reader before/after the test.""" + cmd = ("pip", "uninstall", "shelf-reader", "-y") + subprocess.run(cmd) + yield + subprocess.run(cmd) + + +@pytest.fixture +def python_content_summary(python_repo_api_client, python_repo_version_api_client): + """Get a summary of the repository version's content.""" + def _gen_summary(repository_version=None, repository=None, version=None): + if repository_version is None: + repo_href = get_href(repository) + if version: + repo_ver_href = f"{repo_href}versions/{version}/" + else: + repo_ver_href = python_repo_api_client.read(repo_href).latest_version_href + else: + repo_ver_href = get_href(repository_version) + return python_repo_version_api_client.read(repo_ver_href).content_summary + + yield _gen_summary + + +def get_href(item): + """Tries to get the href from the given item, whether it is a string or object.""" + return item if isinstance(item, str) else item.pulp_href diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index d37c8f07..5592dc87 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -155,6 +155,7 @@ "latest_3": 49, "sdist": 27, "bdist_wheel": 63, + "multi": 33, # keep_latest=1, package_types="bdist_wheel", prereleases=False } DJANGO_LATEST_3 = 4 # latest version has 2 dists, each other has 1 From feb697722d97959c4be557c1cee058477861ef68 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 27 Jun 2024 17:59:34 +0000 Subject: [PATCH 17/84] Release 3.11.2 --- .bumpversion.cfg | 2 +- CHANGES.md | 10 ++++++++++ CHANGES/691.bugfix | 1 - docs/conf.py | 4 ++-- pulp_python/app/__init__.py | 2 +- setup.py | 2 +- 6 files changed, 15 insertions(+), 6 deletions(-) delete mode 100644 CHANGES/691.bugfix diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ade32719..7851f4cb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.11.2.dev +current_version = 3.11.2 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/CHANGES.md b/CHANGES.md index 84412881..1e7464ea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,16 @@ [//]: # (towncrier release notes start) +## 3.11.2 (2024-06-27) {: #3.11.2 } + + +#### Bugfixes {: #3.11.2-bugfix } + +- Fixed the `package_types` filter breaking other remote filters. + [#691](https://github.com/pulp/pulp_python/issues/691) + +--- + ## 3.11.1 (2024-04-11) {: #3.11.1 } ### Bugfixes diff --git a/CHANGES/691.bugfix b/CHANGES/691.bugfix deleted file mode 100644 index 448af729..00000000 --- a/CHANGES/691.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed the `package_types` filter breaking other remote filters. diff --git a/docs/conf.py b/docs/conf.py index 5aa1fb18..98d942d1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.11.2.dev" +version = "3.11.2" # The full version, including alpha/beta/rc tags. -release = "3.11.2.dev" +release = "3.11.2" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 0e370115..2af145ba 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.2.dev" + version = "3.11.2" python_package_name = "pulp-python" diff --git a/setup.py b/setup.py index 168fb600..9cc2a4d6 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.2.dev", + version="3.11.2", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From e904a646ae0f4cc387cb60fc3a1ab1caf90ca734 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 27 Jun 2024 17:59:34 +0000 Subject: [PATCH 18/84] =?UTF-8?q?Bump=20version:=203.11.2=20=E2=86=92=203.?= =?UTF-8?q?11.3.dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulp_python/app/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7851f4cb..06449aac 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.11.2 +current_version = 3.11.3.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 98d942d1..5391b933 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.11.2" +version = "3.11.3.dev" # The full version, including alpha/beta/rc tags. -release = "3.11.2" +release = "3.11.3.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 2af145ba..79ce0fcf 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.2" + version = "3.11.3.dev" python_package_name = "pulp-python" diff --git a/setup.py b/setup.py index 9cc2a4d6..70de72c2 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.2", + version="3.11.3.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From de344a7b13f06b55dd28cac878137c682a7b38c1 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 14 Jul 2024 02:43:17 +0000 Subject: [PATCH 19/84] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 4 + .ci/assets/ci_constraints.txt | 7 + .ci/scripts/calc_constraints.py | 116 ++++++++ .ci/scripts/calc_deps_lowerbounds.py | 34 --- .ci/scripts/changelog.py | 70 ----- .ci/scripts/tweet.py | 16 -- .ci/scripts/update_ci_branches.py | 25 -- .github/template_gitref | 2 +- .github/workflows/build.yml | 70 ++++- .github/workflows/changelog.yml | 58 ---- .github/workflows/nightly.yml | 52 ---- .github/workflows/publish.yml | 176 +----------- .github/workflows/scripts/before_install.sh | 6 +- ...ython_client.sh => build_python_client.sh} | 33 +-- .../workflows/scripts/build_ruby_client.sh | 35 +++ .../scripts/create_release_from_tag.sh | 11 - .github/workflows/scripts/docs-publisher.py | 262 ------------------ .github/workflows/scripts/install.sh | 5 + .../workflows/scripts/install_ruby_client.sh | 42 --- .github/workflows/scripts/post_docs_test.sh | 9 - .github/workflows/scripts/publish_docs.sh | 48 ---- .github/workflows/scripts/release.py | 184 ------------ .github/workflows/scripts/script.sh | 126 ++++----- .github/workflows/scripts/update_ci.sh | 31 --- .github/workflows/test.yml | 73 ++--- CHANGES/.TEMPLATE.rst | 47 ---- template_config.yml | 7 +- 27 files changed, 339 insertions(+), 1210 deletions(-) create mode 100644 .ci/assets/ci_constraints.txt create mode 100755 .ci/scripts/calc_constraints.py delete mode 100755 .ci/scripts/calc_deps_lowerbounds.py delete mode 100755 .ci/scripts/changelog.py delete mode 100755 .ci/scripts/tweet.py delete mode 100755 .ci/scripts/update_ci_branches.py delete mode 100644 .github/workflows/changelog.yml rename .github/workflows/scripts/{install_python_client.sh => build_python_client.sh} (51%) create mode 100755 .github/workflows/scripts/build_ruby_client.sh delete mode 100755 .github/workflows/scripts/create_release_from_tag.sh delete mode 100755 .github/workflows/scripts/docs-publisher.py delete mode 100755 .github/workflows/scripts/install_ruby_client.sh delete mode 100644 .github/workflows/scripts/post_docs_test.sh delete mode 100755 .github/workflows/scripts/publish_docs.sh delete mode 100755 .github/workflows/scripts/release.py delete mode 100755 .github/workflows/scripts/update_ci.sh delete mode 100644 CHANGES/.TEMPLATE.rst diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 338ff697..6cf81f25 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -15,6 +15,9 @@ RUN pip3 install {%- endif -%} {%- for item in plugins -%} {{ " " }}{{ item.source }} +{%- if item.upperbounds | default(false) -%} +{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt +{%- endif -%} {%- if item.lowerbounds | default(false) -%} {{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt {%- endif -%} @@ -22,6 +25,7 @@ RUN pip3 install {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt new file mode 100644 index 00000000..2617a408 --- /dev/null +++ b/.ci/assets/ci_constraints.txt @@ -0,0 +1,7 @@ +# Pulpcore versions without the openapi command do no longer work in the CI +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* + + + +tablib!=3.6.0 +# 3.6.0: This release introduced a regression removing the "html" optional dependency. diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py new file mode 100755 index 00000000..3d00b040 --- /dev/null +++ b/.ci/scripts/calc_constraints.py @@ -0,0 +1,116 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import fileinput +import urllib.request +import sys +from packaging.requirements import Requirement +from packaging.version import Version +import yaml + + +CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" + + +def fetch_pulpcore_upper_bound(requirement): + with urllib.request.urlopen(CORE_TEMPLATE_URL) as f: + template = yaml.safe_load(f.read()) + supported_versions = template["supported_release_branches"] + supported_versions.append(template["latest_release_branch"]) + applicable_versions = sorted( + requirement.specifier.filter((Version(v) for v in supported_versions)) + ) + if len(applicable_versions) == 0: + raise Exception("No supported pulpcore version in required range.") + return f"{requirement.name}~={applicable_versions[-1]}" + + +def split_comment(line): + split_line = line.split("#", maxsplit=1) + try: + comment = " # " + split_line[1].strip() + except IndexError: + comment = "" + return split_line[0].strip(), comment + + +def to_upper_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + if requirement.name == "pulpcore": + # An exception to allow for pulpcore deprecation policy. + return fetch_pulpcore_upper_bound(requirement) + for spec in requirement.specifier: + if spec.operator == "~=": + return f"# NO BETTER CONSTRAINT: {req}" + if spec.operator == "<=": + operator = "==" + max_version = spec.version + return f"{requirement.name}{operator}{max_version}" + if spec.operator == "<": + operator = "~=" + version = Version(spec.version) + if version.micro != 0: + max_version = f"{version.major}.{version.minor}.{version.micro-1}" + elif version.minor != 0: + max_version = f"{version.major}.{version.minor-1}" + else: + return f"# NO BETTER CONSTRAINT: {req}" + return f"{requirement.name}{operator}{max_version}" + return f"# NO UPPER BOUND: {req}" + + +def to_lower_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + for spec in requirement.specifier: + if spec.operator == ">=": + if requirement.name == "pulpcore": + # Currently an exception to allow for pulpcore bugfix releases. + # TODO Semver libraries should be allowed too. + operator = "~=" + else: + operator = "==" + min_version = spec.version + return f"{requirement.name}{operator}{min_version}" + return f"# NO LOWER BOUND: {req}" + + +def main(): + """Calculate constraints for the lower bound of dependencies where possible.""" + parser = argparse.ArgumentParser( + prog=sys.argv[0], + description="Calculate constraints for the lower or upper bound of dependencies where " + "possible.", + ) + parser.add_argument("-u", "--upper", action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true") + parser.add_argument("filename", nargs="*") + args = parser.parse_args() + + with fileinput.input(files=args.filename) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) + else: + req, comment = split_comment(line) + if args.upper: + new_req = to_upper_bound(req) + else: + new_req = to_lower_bound(req) + print(new_req + comment) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/calc_deps_lowerbounds.py b/.ci/scripts/calc_deps_lowerbounds.py deleted file mode 100755 index 2bb48e25..00000000 --- a/.ci/scripts/calc_deps_lowerbounds.py +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -from packaging.requirements import Requirement - - -def main(): - """Calculate the lower bound of dependencies where possible.""" - with open("requirements.txt") as req_file: - for line in req_file: - try: - requirement = Requirement(line) - except ValueError: - print(line.strip()) - else: - for spec in requirement.specifier: - if spec.operator == ">=": - if requirement.name == "pulpcore": - operator = "~=" - else: - operator = "==" - min_version = str(spec)[2:] - print(f"{requirement.name}{operator}{min_version}") - break - else: - print(line.strip()) - - -if __name__ == "__main__": - main() diff --git a/.ci/scripts/changelog.py b/.ci/scripts/changelog.py deleted file mode 100755 index 3f061fad..00000000 --- a/.ci/scripts/changelog.py +++ /dev/null @@ -1,70 +0,0 @@ -import re -import os -import requests -from packaging.version import Version -from git import Repo - -repo = Repo(os.getcwd()) -heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp_python.git").split("\n") -branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])] -branches.sort(key=lambda ver: Version(ver), reverse=True) - - -def get_changelog(branch): - """ - Get changelog file for a given branch. - - """ - return requests.get( - f"https://raw.githubusercontent.com/pulp/pulp_python/{branch}/CHANGES.rst" - ).text - - -def get_changelog_releases(changelog): - """ - Get all versions in changelog. - - """ - versions = re.findall( - r"([0-9]+)\.([0-9]+)\.([0-9][0-9ab]*) \([0-9]{4}-[0-9]{2}-[0-9]{2}\)", changelog - ) - return {".".join(v) for v in versions} - - -def get_changelog_entry(changelog, version): - """ - Get changelog entry for a given version. - - """ - entries = changelog.split(f"{version} (")[1].split("=====\n") - header = f"{version} ({entries[0]}=====\n" - text = "\n\n\n".join(entries[1].split("\n\n\n")[0:-1]) - return header + text + "\n\n\n" - - -main_changelog = get_changelog("main") -main_entries = get_changelog_releases(main_changelog) -entries_list = list(main_entries) -to_add = {} -for branch in branches: - changelog = get_changelog(branch) - entries = get_changelog_releases(changelog) - for entry in entries.difference(main_entries): - description = get_changelog_entry(changelog, entry) - entries_list.append(entry) - print(description) - to_add[entry] = description - -entries_list.sort(key=lambda ver: Version(ver), reverse=True) -for version in sorted(to_add, key=lambda ver: Version(ver)): - next_version = entries_list[entries_list.index(version) + 1] - new_changelog = main_changelog.split(f"{next_version} (")[0] + to_add[version] - new_changelog = new_changelog + f"{next_version} (" - new_changelog = new_changelog + main_changelog.split(f"{next_version} (")[1] - main_changelog = new_changelog - -with open("CHANGES.rst", "w") as f: - f.write(main_changelog) - -if to_add: - repo.git.commit("-m", "Update Changelog\n\n[noissue]", "CHANGES.rst") diff --git a/.ci/scripts/tweet.py b/.ci/scripts/tweet.py deleted file mode 100755 index 5033cf44..00000000 --- a/.ci/scripts/tweet.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import sys -from tweepy import Client - -release_version = sys.argv[1] -if release_version.endswith(".0"): - client = Client( - consumer_key=os.getenv("TWITTER_API_KEY"), - consumer_secret=os.getenv("TWITTER_API_KEY_SECRET"), - access_token=os.getenv("TWITTER_ACCESS_TOKEN"), - access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"), - ) - link = "https://docs.pulpproject.org/pulp_python/changes.html" - msg = f"pulp_python-{release_version} - Check out for more details: {link}" - release_msg = f"Hey! We've just released {msg}" - client.create_tweet(text=release_msg) diff --git a/.ci/scripts/update_ci_branches.py b/.ci/scripts/update_ci_branches.py deleted file mode 100755 index c82e2a07..00000000 --- a/.ci/scripts/update_ci_branches.py +++ /dev/null @@ -1,25 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import os -import sys -import requests - -branches = sys.argv[1:] - -headers = { - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github.v3+json", -} - -github_api = "https://api.github.com" -workflow_path = "/actions/workflows/update_ci.yml/dispatches" -url = f"{github_api}/repos/pulp/pulp_python{workflow_path}" - -for branch in branches: - print(f"Updating {branch}") - requests.post(url, headers=headers, json={"ref": branch}) diff --git a/.github/template_gitref b/.github/template_gitref index 9122ea22..c16f42b6 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-339-gf0d923e +2021.08.26-354-g82d22de diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ed57631..d1fcf2e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,18 +23,30 @@ jobs: with: fetch-depth: 1 path: "pulp_python" + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel + pip install packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Build package" run: | python3 setup.py sdist bdist_wheel --python-tag py3 twine check dist/* + - name: "Install built packages" + run: | + pip install dist/pulp_python-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt + - name: "Generate api specs" + run: | + pulpcore-manager openapi --file "api.json" + pulpcore-manager openapi --bindings --component "python" --file "python-api.json" - name: "Upload Package whl" uses: "actions/upload-artifact@v4" with: @@ -43,3 +55,59 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true + - name: "Upload API specs" + uses: "actions/upload-artifact@v4" + with: + name: "api_spec" + path: | + pulp_python/api.json + pulp_python/python-api.json + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Python bindings packages" + run: | + .github/workflows/scripts/build_python_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v4" + with: + name: "python-client.tar" + path: | + pulp_python/python-python-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload python client docs" + uses: "actions/upload-artifact@v4" + with: + name: "python-client-docs.tar" + path: | + pulp_python/python-python-client-docs.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Ruby bindings packages" + run: | + .github/workflows/scripts/build_ruby_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload Ruby client" + uses: "actions/upload-artifact@v4" + with: + name: "ruby-client.tar" + path: | + pulp_python/python-ruby-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true +... diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml deleted file mode 100644 index 9450eb09..00000000 --- a/.github/workflows/changelog.yml +++ /dev/null @@ -1,58 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - ---- -name: "Python changelog update" -on: - push: - branches: - - "main" - paths: - - "CHANGES.rst" - - "CHANGES.md" - workflow_dispatch: - -jobs: - - update-changelog: - runs-on: "ubuntu-latest" - strategy: - fail-fast: false - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install -r doc_requirements.txt - echo ::endgroup:: - - - name: "Fake api schema" - run: | - mkdir -p docs/_build/html - echo "{}" > docs/_build/html/api.json - mkdir -p docs/_static - echo "{}" > docs/_static/api.json - - name: "Build Docs" - run: | - make diagrams html - working-directory: "./docs" - env: - PULP_CONTENT_ORIGIN: "http://localhost/" - - - name: "Publish changlog to pulpproject.org" - run: | - .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} - env: - PULP_DOCS_KEY: "${{ secrets.PULP_DOCS_KEY }}" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5b9b39ac..eeb6ae45 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -65,55 +65,3 @@ jobs: branch: "changelog/update" delete-branch: true path: "pulp_python" - - publish: - runs-on: ubuntu-latest - needs: test - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_python" - - - uses: actions/download-artifact@v4 - with: - name: "plugin_package" - path: "pulp_python/dist/" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1' - echo ::endgroup:: - - - name: "Set environment variables" - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Download built docs - uses: actions/download-artifact@v4 - with: - name: "docs.tar" - path: "pulp_python" - - - name: Download Python client docs - uses: actions/download-artifact@v4 - with: - name: "python-client-docs.tar" - path: "pulp_python" - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar -C ./docs - .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1c225e73..274490d9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,131 +19,10 @@ defaults: jobs: build: uses: "./.github/workflows/build.yml" - - build-bindings-docs: - needs: - - "build" - runs-on: "ubuntu-latest" - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: "docs" - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_python" - - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - repository: "pulp/pulp-openapi-generator" - path: "pulp-openapi-generator" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - uses: "actions/download-artifact@v4" - with: - name: "plugin_package" - path: "pulp_python/dist/" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: "Before Install" - run: | - .github/workflows/scripts/before_install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install" - run: | - .github/workflows/scripts/install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - - name: "Upload python client packages" - uses: "actions/upload-artifact@v4" - with: - name: "python-client.tar" - path: | - pulp_python/python-python-client.tar - if-no-files-found: "error" - overwrite: true - - - name: "Upload python client docs" - uses: "actions/upload-artifact@v4" - with: - name: "python-client-docs.tar" - path: | - pulp_python/python-python-client-docs.tar - if-no-files-found: "error" - overwrite: true - - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v4" - with: - name: "ruby-client.tar" - path: | - pulp_python/python-ruby-client.tar - if-no-files-found: "error" - overwrite: true - - name: Build docs - run: | - export DJANGO_SETTINGS_MODULE=pulpcore.app.settings - export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py - make -C docs/ PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" diagrams html - tar -cvf docs/docs.tar docs/_build - - - name: "Upload built docs" - uses: actions/upload-artifact@v4 - with: - name: "docs.tar" - path: "pulp_python/docs/docs.tar" - if-no-files-found: "error" - overwrite: true - - - name: "Logs" - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" publish-package: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -181,7 +60,7 @@ jobs: publish-python-bindings: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -224,7 +103,7 @@ jobs: publish-ruby-bindings: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -258,61 +137,14 @@ jobs: - name: "Publish client to rubygems" run: | bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} - publish-docs: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_python" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install 'packaging~=21.3' requests - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Download built docs" - uses: "actions/download-artifact@v4" - with: - name: "docs.tar" - path: "pulp_python/" - - - name: "Download Python client docs" - uses: "actions/download-artifact@v4" - with: - name: "python-client-docs.tar" - path: "pulp_python/" - - - name: "Publish docs to pulpproject.org" - run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }} create-gh-release: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" - "publish-package" - "publish-python-bindings" - "publish-ruby-bindings" - - "publish-docs" steps: - name: "Create release on GitHub" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 5abc9c98..80be9043 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -65,9 +65,11 @@ then exit $s fi +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt +fi if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt - sed -i 's/\[.*\]//g' lowerbounds_constraints.txt + python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/build_python_client.sh similarity index 51% rename from .github/workflows/scripts/install_python_client.sh rename to .github/workflows/scripts/build_python_client.sh index 8c1fe8f0..5bf6e16d 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -1,5 +1,8 @@ #!/bin/bash +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -python-client.tar and -python-client-docs.tar file in the plugins root directory. + # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use @@ -12,36 +15,20 @@ set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -source .github/workflows/scripts/utils.sh - -PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "python" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - pushd ../pulp-openapi-generator -rm -rf pulp_python-client +rm -rf "pulp_python-client" -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=python" - USE_LOCAL_API_JSON=1 ./generate.sh pulp_python python "$VERSION" -else - ./generate.sh pulp_python python "$VERSION" -fi +# We need to copy that over to be visible in the container. +cp "../pulp_python/python-api.json" . +./gen-client.sh "python-api.json" "python" python "pulp_python" pushd pulp_python-client python setup.py sdist bdist_wheel --python-tag py3 -twine check "dist/pulp_python_client-$VERSION-py3-none-any.whl" -twine check "dist/pulp_python-client-$VERSION.tar.gz" +twine check "dist/pulp_python_client-"*"-py3-none-any.whl" +twine check "dist/pulp_python-client-"*".tar.gz" -cmd_prefix pip3 install "/root/pulp-openapi-generator/pulp_python-client/dist/pulp_python_client-${VERSION}-py3-none-any.whl" -tar cvf ../../pulp_python/python-python-client.tar ./dist +tar cvf "../../pulp_python/python-python-client.tar" ./dist find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; find ./docs/* -exec sed -i 's/README//g' {} \; diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh new file mode 100755 index 00000000..a4abec1d --- /dev/null +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -ruby-client.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +rm -rf "pulp_python-client" + +# We need to copy that over to be visible in the container. +#cp "../pulp_python/python-api.json" . +#./gen-client.sh "python-api.json" "python" ruby "pulp_python" + +# ------------- +# The generator still needs to have it called api.json at this time... +cp "../pulp_python/api.json" . +./gen-client.sh "api.json" "python" ruby "pulp_python" +# ------------- + +pushd pulp_python-client +gem build pulp_python_client +tar cvf "../../pulp_python/python-ruby-client.tar" "./pulp_python_client-"*".gem" +popd +popd diff --git a/.github/workflows/scripts/create_release_from_tag.sh b/.github/workflows/scripts/create_release_from_tag.sh deleted file mode 100755 index 328454e0..00000000 --- a/.github/workflows/scripts/create_release_from_tag.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -e - -curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/releases \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "tag_name": "$1", - "name": "$1" -} -EOF diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py deleted file mode 100755 index 61ae6e47..00000000 --- a/.github/workflows/scripts/docs-publisher.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import subprocess -import os -import re -from shutil import rmtree -import tempfile -import requests -import json -from packaging import version - -WORKING_DIR = os.environ["WORKSPACE"] - -VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" -RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" - -USERNAME = "doc_builder_pulp_python" -HOSTNAME = "8.43.85.236" - -SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_python/" - - -def make_directory_with_rsync(remote_paths_list): - """ - Ensure the remote directory path exists. - - :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the - remote. - :type remote_paths_list: a list of strings, with each string representing a directory. - """ - try: - tempdir_path = tempfile.mkdtemp() - cwd = os.getcwd() - os.chdir(tempdir_path) - os.makedirs(os.sep.join(remote_paths_list)) - remote_path_arg = "%s@%s:%s%s" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - remote_paths_list[0], - ) - local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep - rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command) - if exit_code != 0: - raise RuntimeError("An error occurred while creating remote directories.") - finally: - rmtree(tempdir_path) - os.chdir(cwd) - - -def ensure_dir(target_dir, clean=True): - """ - Ensure that the directory specified exists and is empty. - - By default this will delete the directory if it already exists - - :param target_dir: The directory to process - :type target_dir: str - :param clean: Whether or not the directory should be removed and recreated - :type clean: bool - """ - if clean: - rmtree(target_dir, ignore_errors=True) - try: - os.makedirs(target_dir) - except OSError: - pass - - -def main(): - """ - Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--build-type", required=True, help="Build type: nightly, tag or changelog." - ) - parser.add_argument("--branch", required=True, help="Branch or tag name.") - opts = parser.parse_args() - - build_type = opts.build_type - - branch = opts.branch - - publish_at_root = False - - # rsync the docs - print("rsync the docs") - docs_directory = os.sep.join([WORKING_DIR, "docs"]) - local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep - if build_type == "nightly": - # This is a nightly build - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - make_directory_with_rsync(["en", branch, build_type]) - rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: - # Only publish docs at the root if this is the latest version - r = requests.get("https://pypi.org/pypi/pulp-python/json") - latest_version = version.parse(json.loads(r.text)["info"]["version"]) - docs_version = version.parse(branch) - # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. - # It doesn't prevent incorrect docs being published at root if 2 releases are done close - # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately - # after. - if docs_version >= latest_version: - publish_at_root = True - # Post releases should use the x.y.z part of the version string to form a path - if "post" in branch: - branch = ".".join(branch.split(".")[:-1]) - - # This is a GA build. - # publish to the root of docs.pulpproject.org - if publish_at_root: - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "en", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - # This is a pre-release - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "nightly", - "--exclude", - "testing", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "changelog": - if branch != "main": - raise RuntimeError("Can only publish CHANGELOG from main") - # Publish the CHANGELOG from main branch at the root directory - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - changelog_local_path_arg = os.path.join(local_path_arg, "changes.html") - rsync_command = [ - "rsync", - "-vzh", - "--omit-dir-times", - changelog_local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - raise RuntimeError("Build type must be either 'nightly', 'tag' or 'changelog'.") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 356eba1e..af87839e 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -57,6 +57,11 @@ if [[ -f ../../ci_requirements.txt ]]; then ci_requirements: true VARSYAML fi +if [ "$TEST" = "pulp" ]; then + cat >> vars/main.yaml << VARSYAML + upperbounds: true +VARSYAML +fi if [ "$TEST" = "lowerbounds" ]; then cat >> vars/main.yaml << VARSYAML lowerbounds: true diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh deleted file mode 100755 index 7ab198a5..00000000 --- a/.github/workflows/scripts/install_ruby_client.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -source .github/workflows/scripts/utils.sh - -PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24%7BPULP_URL%3A-https%3A%2F%2Fpulp%7D" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "python" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - -pushd ../pulp-openapi-generator -rm -rf pulp_python-client - -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=python" - USE_LOCAL_API_JSON=1 ./generate.sh pulp_python ruby "$VERSION" -else - ./generate.sh pulp_python ruby "$VERSION" -fi - -pushd pulp_python-client -gem build pulp_python_client -gem install --both "./pulp_python_client-$VERSION.gem" -tar cvf ../../pulp_python/python-ruby-client.tar "./pulp_python_client-$VERSION.gem" -popd -popd diff --git a/.github/workflows/scripts/post_docs_test.sh b/.github/workflows/scripts/post_docs_test.sh deleted file mode 100644 index 2178e5a8..00000000 --- a/.github/workflows/scripts/post_docs_test.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env sh - -export BASE_ADDR=https://pulp -export CONTENT_ADDR=https://pulp - -pip install -r doc_requirements.txt - -cd docs/_scripts/ -bash ./quickstart.sh diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh deleted file mode 100755 index db944810..00000000 --- a/.github/workflows/scripts/publish_docs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -mkdir ~/.ssh -touch ~/.ssh/pulp-infra -chmod 600 ~/.ssh/pulp-infra -echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra - -echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts -chmod 644 ~/.ssh/known_hosts - -export PYTHONUNBUFFERED=1 -export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export WORKSPACE=$PWD - -# start the ssh agent -eval "$(ssh-agent -s)" -ssh-add ~/.ssh/pulp-infra - -python3 .github/workflows/scripts/docs-publisher.py --build-type "$1" --branch "$2" - -if [[ "$GITHUB_WORKFLOW" == "Python changelog update" ]]; then - # Do not build bindings docs on changelog update - exit -fi - -mkdir -p ../python-bindings -tar -xvf python-python-client-docs.tar --directory ../python-bindings -pushd ../python-bindings - -# publish to docs.pulpproject.org/pulp_python_client -rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/ - -# publish to docs.pulpproject.org/pulp_python_client/en/{release} -rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/en/"$2" -popd diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py deleted file mode 100755 index 04d2e0d9..00000000 --- a/.github/workflows/scripts/release.py +++ /dev/null @@ -1,184 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import re -import os -import textwrap -import requests -import subprocess - -from git import Repo -from pathlib import Path - - -def get_package_from_pypi(version, plugin_path): - """ - Download a package from PyPI. - - :param version: version of the package to download from PyPI - :return: True/False if download was successful - """ - os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) - r = requests.get(f"https://pypi.org/pypi/pulp-python/{version}/json") - if r.status_code == 200: - metadata = r.json() - for url_data in metadata["urls"]: - filename = url_data["filename"] - r2 = requests.get(url_data["url"]) - if r2.status_code != 200: - raise RuntimeError(f"Failed to download released artifact {filename}") - with open(os.path.join(plugin_path, "dist", filename), "wb") as f: - f.write(r2.content) - return True - return False - - -def create_release_commits(repo, release_version, plugin_path): - """Build changelog, set version, commit, bump to next dev version, commit.""" - issues_to_close = set() - for filename in Path(f"{plugin_path}/CHANGES").rglob("*"): - if filename.stem.isdigit(): - issue = filename.stem - issues_to_close.add(issue) - - issues = ",".join(issues_to_close) - # First commit: changelog - os.system(f"towncrier --yes --version {release_version}") - git = repo.git - git.add("CHANGES.rst") - git.add("CHANGES/*") - git.commit("-m", f"Add changelog for {release_version}\n\n[noissue]") - - # Second commit: release version - os.system("bump2version release --allow-dirty") - - git.add(f"{plugin_path}/pulp_python/*") - git.add(f"{plugin_path}/docs/conf.py") - git.add(f"{plugin_path}/setup.py") - git.add(f"{plugin_path}/requirements.txt") - git.add(f"{plugin_path}/.bumpversion.cfg") - git.commit("-m", f"Release {release_version}\nGH Issues: {issues}\n\n[noissue]") - sha = repo.head.object.hexsha - short_sha = git.rev_parse(sha, short=7) - - os.system("bump2version patch --allow-dirty") - - new_dev_version = None - with open(f"{plugin_path}/setup.py") as fp: - for line in fp.readlines(): - if "version=" in line: - new_dev_version = re.split("\"|'", line)[1] - if not new_dev_version: - raise RuntimeError("Could not detect new dev version ... aborting.") - - git.add(f"{plugin_path}/pulp_python/*") - git.add(f"{plugin_path}/docs/conf.py") - git.add(f"{plugin_path}/setup.py") - git.add(f"{plugin_path}/requirements.txt") - git.add(f"{plugin_path}/.bumpversion.cfg") - git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]") - print(f"Release commit == {short_sha}") - print(f"All changes were committed on branch: release_{release_version}") - return sha - - -def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): - """Create a tag if one is needed and build a package if one is not on PyPI.""" - # Remove auth header config - with repo.config_writer() as conf: - conf.remove_section('http "https://github.com/"') - conf.release() - - # Determine if a tag exists and if it matches the specified commit sha - tag = None - for existing_tag in repo.tags: - if existing_tag.name == desired_tag: - if existing_tag.commit.hexsha == commit_sha: - tag = existing_tag - else: - raise RuntimeError( - f"The '{desired_tag}' tag already exists, but the commit sha does not match " - f"'{commit_sha}'." - ) - - # Create a tag if one does not exist - if not tag: - tag = repo.create_tag(desired_tag, ref=commit_sha) - - # Checkout the desired tag and reset the tree - repo.head.reference = tag.commit - repo.head.reset(index=True, working_tree=True) - - # Check if Package is available on PyPI - if not get_package_from_pypi(tag.name, plugin_path): - os.system("python3 setup.py sdist bdist_wheel --python-tag py3") - - -def main(): - helper = textwrap.dedent( - """\ - Start the release process. - - Example: - setup.py on plugin before script: - version="2.0.0.dev" - - $ python .ci/scripts/release.py - - setup.py on plugin after script: - version="2.0.1.dev" - - - """ - ) - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, description=helper - ) - - parser.add_argument( - "release_version", - type=str, - help="The version string for the release.", - ) - - args = parser.parse_args() - - release_version_arg = args.release_version - - release_path = os.path.dirname(os.path.abspath(__file__)) - plugin_path = release_path.split("/.github")[0] - - output = subprocess.check_output(["bump2version", "--dry-run", "--list", "release"]) - release_version = re.findall(r"\nnew_version=([0-9.]*)\n", output.decode())[0] - - print(f"\n\nRepo path: {plugin_path}") - repo = Repo(plugin_path) - - release_commit = None - if release_version != release_version_arg: - # Look for a commit with the requested release version - for commit in repo.iter_commits(): - if f"Release {release_version_arg}\n" in commit.message: - release_commit = commit - release_version = release_version_arg - break - if not release_commit: - raise RuntimeError( - f"The release version {release_version_arg} does not match the .dev version at " - "HEAD. A release commit for such version does not exist." - ) - - if not release_commit: - release_commit_sha = create_release_commits(repo, release_version, plugin_path) - else: - release_commit_sha = release_commit.hexsha - create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 131ad0dc..0e9c935d 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -16,7 +16,6 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. source .github/workflows/scripts/utils.sh export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh -export POST_DOCS_TEST=$PWD/.github/workflows/scripts/post_docs_test.sh export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh # Needed for both starting the service and building the docs. @@ -31,15 +30,7 @@ if [[ "$TEST" = "docs" ]]; then if [[ "$GITHUB_WORKFLOW" == "Python CI" ]]; then towncrier build --yes --version 4.0.0.ci fi - # Legacy Docs Build - cd docs - make PULP_URL="iframe.php?url=https%3A%2F%2Fgithub.com%2F%24PULP_URL" diagrams html - tar -cvf docs.tar ./_build - cd .. - - if [ -f "$POST_DOCS_TEST" ]; then - source "$POST_DOCS_TEST" - fi + pulp-docs build exit fi @@ -52,75 +43,74 @@ password password # Some commands like ansible-galaxy specifically require 600 cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -# Generate and install binding +# Generate bindings +################### + +echo "::group::Generate bindings" + +touch bindings_requirements.txt pushd ../pulp-openapi-generator -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then # Use app_label to generate api.json and package to produce the proper package name. - if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] + # Workaround: Domains are not supported by the published bindings. + # Sadly: Different pulpcore-versions aren't either... + # So we exclude the prebuilt ones only for domains disabled. + if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ] then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done + BUILT_CLIENTS="" else - # Sadly: Different pulpcore-versions aren't either... - for item in $(jq -r '.versions[]| select(.component!="python")| tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done + BUILT_CLIENTS=" python " fi -else - # Infer the client name from the package name by replacing "-" with "_". - # Use the component to infer the package name on older versions of pulpcore. - if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] - then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - else - # Sadly: Different pulpcore-versions aren't either... - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="python")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - fi -fi + for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") + do + COMPONENT="$(jq -r '.component' <<<"${ITEM}")" + VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")" + # On older status endpoints, the module was not provided, but the package should be accurate + # there, because we did not merge plugins into pulpcore back then. + MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")" + PACKAGE="${MODULE%%.*}" + cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json" + if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]] + then + rm -rf "./${PACKAGE}-client" + ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" + pushd "${PACKAGE}-client" + python setup.py sdist bdist_wheel --python-tag py3 + popd + else + if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] + then + ls -lR "${PACKAGE}-client/" + echo "Error: Client bindings for ${COMPONENT} not found." + echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing." + exit 1 + fi + fi + echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp_python/bindings_requirements.txt" + done popd -# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. -echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt +echo "::endgroup::" + +echo "::group::Debug bindings diffs" + +# Bindings diff for python +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "python-api.json" > "build-api.json" +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "../pulp-openapi-generator/python-api.json" > "test-api.json" +jsondiff --indent 2 build-api.json test-api.json || true +echo "::endgroup::" + +# Install test requirements +########################### + +# Add a safeguard to make sure the proper versions of the clients are installed. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" @@ -151,7 +141,7 @@ else else cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel'" - fi + fi fi pushd ../pulp-cli pip install -r test_requirements.txt diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh deleted file mode 100755 index b22001b4..00000000 --- a/.github/workflows/scripts/update_ci.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -eu - -if [ ! -f "template_config.yml" ]; then - echo "No template_config.yml detected." - exit 1 -fi - -pushd ../plugin_template -./plugin-template --github pulp_python -popd - -# Check if only gitref file has changed, so no effect on CI workflows. -if [[ $(git diff --name-only) == ".github/template_gitref" ]]; then - echo "No changes detected in github section." - git restore ".github/template_gitref" -fi - -if [[ $(git status --porcelain) ]]; then - git add -A - git commit -m "Update CI files" -m "[noissue]" -else - echo "No updates needed" -fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4738d1c1..afc64d68 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,15 +43,36 @@ jobs: with: python-version: "3.11" - - uses: "actions/download-artifact@v4" + - name: "Download plugin package" + uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_python/dist/" + - name: "Download API specs" + uses: "actions/download-artifact@v4" + with: + name: "api_spec" + path: "pulp_python/" + + - name: "Download client packages" + uses: "actions/download-artifact@v4" + with: + name: "python-client.tar" + path: "pulp_python" + + - name: "Unpack client packages" + working-directory: "pulp-openapi-generator" + run: | + mkdir -p "pulp_python-client" + pushd "pulp_python-client" + tar xvf "../../pulp_python/python-python-client.tar" + popd + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs jq jsonpatch echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: @@ -94,16 +115,6 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - if: "${{ env.TEST == 'pulp' }}" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - name: "Script" run: | .github/workflows/scripts/script.sh @@ -126,43 +137,6 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true - - name: Upload python client packages - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v4 - with: - name: "python-client.tar" - path: "pulp_python/python-python-client.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - - - name: Upload python client docs - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v4 - with: - name: "python-client-docs.tar" - path: "pulp_python/python-python-client-docs.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - - name: Upload Ruby client - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v4 - with: - name: "ruby-client.tar" - path: "pulp_python/python-ruby-client.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - - name: Upload built docs - if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v4 - with: - name: "docs.tar" - path: "pulp_python/docs/docs.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - name: "Logs" if: always() @@ -175,3 +149,4 @@ jobs: docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +... diff --git a/CHANGES/.TEMPLATE.rst b/CHANGES/.TEMPLATE.rst deleted file mode 100644 index 49c2305d..00000000 --- a/CHANGES/.TEMPLATE.rst +++ /dev/null @@ -1,47 +0,0 @@ -{% if render_title %} -{% if versiondata.name %} -{{ versiondata.name }} {{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.name + versiondata.version + versiondata.date)|length + 4)}} -{% else %} -{{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.version + versiondata.date)|length + 3)}} -{% endif %} -{% endif %} -{% for section, _ in sections.items() %} -{% set underline = underlines[0] %}{% if section %}{{section}} -{{ underline * section|length }}{% set underline = underlines[1] %} - -{% endif %} - -{% if sections[section] %} -{% for category, val in definitions.items() if category in sections[section]%} -{{ definitions[category]['name'] }} -{{ underline * definitions[category]['name']|length }} - -{% if definitions[category]['showcontent'] %} -{% for text, values in sections[section][category].items() %} -- {{ text }} - {{ values|join(',\n ') }} -{% endfor %} - -{% else %} -- {{ sections[section][category]['']|join(', ') }} - -{% endif %} -{% if sections[section][category]|length == 0 %} -No significant changes. - -{% else %} -{% endif %} - -{% endfor %} -{% else %} -No significant changes. - - -{% endif %} -{% endfor %} ----- - - - diff --git a/template_config.yml b/template_config.yml index 32c874a1..57514cfc 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-326-ge5addc7 +# generated with plugin_template@2021.08.26-354-g82d22de api_root: /pulp/ black: false @@ -23,7 +23,6 @@ disabled_redis_runners: [] doc_requirements_from_pulpcore: true docker_fixtures: false docs_test: true -extra_docs_requirements: [] flake8: true flake8_ignore: [] github_org: pulp @@ -32,6 +31,7 @@ kanban: true latest_release_branch: null lint_requirements: true noissue_marker: '[noissue]' +os_required_packages: [] parallel_test_workers: 8 plugin_app_label: python plugin_default_branch: main @@ -41,7 +41,6 @@ plugins: name: pulp_python post_job_template: null pre_job_template: null -publish_docs_to_pulpprojectdotorg: true pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} @@ -73,6 +72,4 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true -use_legacy_docs: true -use_unified_docs: false From eb59b3593a75a21a29d7a268b433b1572c4eb350 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 14 Jul 2024 02:43:17 +0000 Subject: [PATCH 20/84] Bump pulpcore lowerbounds to supported branch [noissue] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bb40f6ad..3d7b2059 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pulpcore>=3.25.0,<3.55 +pulpcore>=3.28,<3.55 pkginfo>=1.8.2,<1.9.7 bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 From 41ead7e9005f16daf68b2aad62c4da22831f0d94 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 23 Jul 2024 09:46:47 +0000 Subject: [PATCH 21/84] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 7 +++++-- .ci/scripts/calc_constraints.py | 2 ++ .github/template_gitref | 2 +- .github/workflows/scripts/build_python_client.sh | 4 +--- .github/workflows/scripts/build_ruby_client.sh | 10 +--------- .github/workflows/test.yml | 1 - template_config.yml | 2 +- 7 files changed, 11 insertions(+), 17 deletions(-) diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 6cf81f25..afe0b118 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -9,7 +9,9 @@ ADD ./{{ item.name }} ./{{ item.name }} # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads # Hacking botocore (https://github.com/boto/botocore/pull/1990) -RUN pip3 install +RUN pip3 install --upgrade pip setuptools wheel && \ + rm -rf /root/.cache/pip && \ + pip3 install {%- if s3_test | default(false) -%} {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} @@ -25,7 +27,8 @@ RUN pip3 install {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ + rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 3d00b040..badca1c9 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -62,6 +62,8 @@ def to_upper_bound(req): max_version = f"{version.major}.{version.minor}.{version.micro-1}" elif version.minor != 0: max_version = f"{version.major}.{version.minor-1}" + elif version.major != 0: + max_version = f"{version.major-1}.0" else: return f"# NO BETTER CONSTRAINT: {req}" return f"{requirement.name}{operator}{max_version}" diff --git a/.github/template_gitref b/.github/template_gitref index c16f42b6..e1ea9541 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-354-g82d22de +2021.08.26-362-g5add0ff diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh index 5bf6e16d..e86ffc29 100755 --- a/.github/workflows/scripts/build_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -18,9 +18,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pushd ../pulp-openapi-generator rm -rf "pulp_python-client" -# We need to copy that over to be visible in the container. -cp "../pulp_python/python-api.json" . -./gen-client.sh "python-api.json" "python" python "pulp_python" +./gen-client.sh "../pulp_python/python-api.json" "python" python "pulp_python" pushd pulp_python-client python setup.py sdist bdist_wheel --python-tag py3 diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh index a4abec1d..6b433edd 100755 --- a/.github/workflows/scripts/build_ruby_client.sh +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -18,15 +18,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pushd ../pulp-openapi-generator rm -rf "pulp_python-client" -# We need to copy that over to be visible in the container. -#cp "../pulp_python/python-api.json" . -#./gen-client.sh "python-api.json" "python" ruby "pulp_python" - -# ------------- -# The generator still needs to have it called api.json at this time... -cp "../pulp_python/api.json" . -./gen-client.sh "api.json" "python" ruby "pulp_python" -# ------------- +./gen-client.sh "../pulp_python/python-api.json" "python" ruby "pulp_python" pushd pulp_python-client gem build pulp_python_client diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index afc64d68..1155aee3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,6 @@ jobs: matrix: env: - TEST: pulp - - TEST: docs - TEST: azure - TEST: s3 - TEST: lowerbounds diff --git a/template_config.yml b/template_config.yml index 57514cfc..49b8b7cc 100644 --- a/template_config.yml +++ b/template_config.yml @@ -22,7 +22,7 @@ deploy_to_pypi: true disabled_redis_runners: [] doc_requirements_from_pulpcore: true docker_fixtures: false -docs_test: true +docs_test: false flake8: true flake8_ignore: [] github_org: pulp From 72f9ef5c79d81afcee2eaab4b6f4de02695f561d Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 28 Jul 2024 02:41:33 +0000 Subject: [PATCH 22/84] Update CI files [noissue] --- .github/template_gitref | 2 +- .github/workflows/kanban.yml | 103 ----------------------------------- template_config.yml | 3 +- 3 files changed, 2 insertions(+), 106 deletions(-) delete mode 100644 .github/workflows/kanban.yml diff --git a/.github/template_gitref b/.github/template_gitref index e1ea9541..9e5cdc02 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-362-g5add0ff +2021.08.26-364-g6f9579c diff --git a/.github/workflows/kanban.yml b/.github/workflows/kanban.yml deleted file mode 100644 index 370f4bee..00000000 --- a/.github/workflows/kanban.yml +++ /dev/null @@ -1,103 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -# Manage issues in a project board using https://github.com/leonsteinhaeuser/project-beta-automations - ---- -name: Kanban -on: - pull_request_target: - issues: - types: - - labeled - - reopened - - assigned - - closed - -env: - free_to_take: Free to take - in_progress: In Progress - needs_review: Needs review - done: Done - -jobs: - # only prio-list labeled items should be added to the board - add-to-project-board: - if: github.event_name == 'issues' && contains(github.event.issue.labels.*.name, 'prio-list') && contains(fromJson('["labeled", "reopened"]'), github.event.action) - runs-on: ubuntu-latest - steps: - - name: Add issue to Free-to-take list - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.free_to_take }} # Target status - - move-to-inprogress: - if: github.event_name == 'issues' && github.event.action == 'assigned' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the In Progress column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.in_progress }} # Target status - - find-linked-issues: - if: github.event_name == 'pull_request_target' - runs-on: ubuntu-latest - name: Find issues linked to a PR - outputs: - linked-issues: ${{ steps.linked-issues.outputs.issues }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Get Linked Issues Action - uses: kin/gh-action-get-linked-issues@v1.0 - id: linked-issues - with: - access-token: ${{ secrets.RELEASE_TOKEN }} - - move-to-needs-review: - if: github.event_name == 'pull_request_target' && contains(fromJson(needs.find-linked-issues.outputs.linked-issues).*.issue.state, 'open') - runs-on: ubuntu-latest - name: Move linked issues to Needs Review - needs: find-linked-issues - strategy: - max-parallel: 3 - matrix: - issues: ${{ fromJSON(needs.find-linked-issues.outputs.linked-issues) }} - steps: - - name: Move to Needs Review - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ matrix.issues.issue.node_id }} - operation_mode: status - status_value: ${{ env.needs_review }} # Target status - - move-to-done: - if: github.event_name == 'issues' && github.event.action == 'closed' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the Done column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.done }} # Target status diff --git a/template_config.yml b/template_config.yml index 49b8b7cc..4dca7ef3 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-354-g82d22de +# generated with plugin_template@2021.08.26-364-g6f9579c api_root: /pulp/ black: false @@ -27,7 +27,6 @@ flake8: true flake8_ignore: [] github_org: pulp issue_tracker: github -kanban: true latest_release_branch: null lint_requirements: true noissue_marker: '[noissue]' From b66adac4cba9443511bc50d9fae975cf3afd8df1 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 11 Aug 2024 02:43:12 +0000 Subject: [PATCH 23/84] Update CI files [noissue] --- .ci/scripts/validate_commit_message.py | 8 +--- .github/template_gitref | 2 +- .github/workflows/ci.yml | 4 ++ .github/workflows/create-branch.yml | 15 ++++--- .github/workflows/docs.yml | 54 ++++++++++++++++++++++++++ .github/workflows/scripts/install.sh | 19 +++------ .github/workflows/scripts/script.sh | 10 +---- .github/workflows/update_ci.yml | 2 +- doc_requirements.txt | 20 ++++------ template_config.yml | 6 +-- 10 files changed, 83 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/docs.yml diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index d6f22c71..9c718a83 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -9,21 +9,17 @@ import sys from pathlib import Path import subprocess - - import os import warnings from github import Github - NO_ISSUE = "[noissue]" CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] +KEYWORDS = ["fixes", "closes"] + sha = sys.argv[1] message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - -KEYWORDS = ["fixes", "closes"] - g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp_python") diff --git a/.github/template_gitref b/.github/template_gitref index 9e5cdc02..ded489e7 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-364-g6f9579c +2021.08.26-377-g1ba8f46 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17800617..db87bfcc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,9 @@ jobs: run: | python .ci/scripts/check_requirements.py + docs: + uses: "./.github/workflows/docs.yml" + lint: uses: "./.github/workflows/lint.yml" @@ -84,6 +87,7 @@ jobs: - "check-commits" - "lint" - "test" + - "docs" if: "always()" steps: - name: "Collect needed jobs results" diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index b207c880..b35c4bef 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -26,6 +26,12 @@ jobs: fetch-depth: 0 path: "pulp_python" + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/plugin_template" + path: "plugin_template" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -33,7 +39,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version jinja2 pyyaml packaging + pip install bump2version packaging -r plugin_template/requirements.txt echo ::endgroup:: - name: "Setting secrets" @@ -71,13 +77,6 @@ jobs: run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + - - name: Checkout plugin template - uses: actions/checkout@v4 - with: - repository: pulp/plugin_template - path: plugin_template - fetch-depth: 0 - - name: Update CI branches in template_config working-directory: plugin_template run: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..692ba626 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,54 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Docs" +on: + workflow_call: + +jobs: + test: + if: "endsWith(github.base_ref, 'main')" + runs-on: "ubuntu-20.04" + defaults: + run: + working-directory: "pulp_python" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Setup cache key" + run: | + git ls-remote https://github.com/pulp/pulp-docs main | tee pulp-docs-main-sha + - uses: "actions/cache@v4" + with: + path: "~/.cache/pip" + key: ${{ runner.os }}-pip-${{ hashFiles('pulp-docs-main-sha') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install -r doc_requirements.txt + echo ::endgroup:: + - name: "Build changelog" + run: | + towncrier build --yes --version 4.0.0.ci + - name: "Build docs" + run: | + pulp-docs build + + no-test: + if: "!endsWith(github.base_ref, 'main')" + runs-on: "ubuntu-20.04" + steps: + - run: | + echo "Skip docs testing on non-main branches." diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index af87839e..9711e5cf 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -21,20 +21,13 @@ PLUGIN_SOURCE="./pulp_python/dist/pulp_python-${PLUGIN_VERSION}-py3-none-any.whl export PULP_API_ROOT="/pulp/" PIP_REQUIREMENTS=("pulp-cli") -if [[ "$TEST" = "docs" || "$TEST" = "publish" ]] -then - PIP_REQUIREMENTS+=("-r" "doc_requirements.txt") - git clone https://github.com/pulp/pulpcore.git ../pulpcore - PIP_REQUIREMENTS+=("psycopg2-binary" "-r" "../pulpcore/doc_requirements.txt") -fi +# This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} -if [[ "$TEST" != "docs" ]] -then - PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" - git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -fi +# Check out the pulp-cli branch matching the installed version. +PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" +git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli cd .ci/ansible/ if [ "$TEST" = "s3" ]; then @@ -128,9 +121,7 @@ if [ "${PULP_API_ROOT:-}" ]; then fi pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" --username "admin" --password "password" -if [[ "$TEST" != "docs" ]]; then - cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" -fi +cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml ansible-playbook start_container.yaml diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 0e9c935d..60284868 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -18,7 +18,7 @@ source .github/workflows/scripts/utils.sh export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh -# Needed for both starting the service and building the docs. +# Needed for starting the service # Gets set in .github/settings.yml, but doesn't seem to inherited by # this script. export DJANGO_SETTINGS_MODULE=pulpcore.app.settings @@ -26,14 +26,6 @@ export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py export PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" -if [[ "$TEST" = "docs" ]]; then - if [[ "$GITHUB_WORKFLOW" == "Python CI" ]]; then - towncrier build --yes --version 4.0.0.ci - fi - pulp-docs build - exit -fi - REPORTED_STATUS="$(pulp status)" echo "machine pulp diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index f34476b7..e51899f4 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -36,7 +36,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install gitpython requests packaging jinja2 pyyaml + pip install gitpython packaging -r plugin_template/requirements.txt echo ::endgroup:: - name: "Configure Git with pulpbot name and email" diff --git a/doc_requirements.txt b/doc_requirements.txt index 925c79a0..6456b7d8 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,14 +1,8 @@ -build -coreapi -django -djangorestframework -django-filter -drf-nested-routers -mistune -plantuml -pyyaml -sphinx -sphinx-rtd-theme -sphinxcontrib-openapi +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template towncrier -twine +pulp-docs @ git+https://github.com/pulp/pulp-docs@main diff --git a/template_config.yml b/template_config.yml index 4dca7ef3..e7b07822 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-364-g6f9579c +# generated with plugin_template@2021.08.26-377-g1ba8f46 api_root: /pulp/ black: false @@ -12,7 +12,6 @@ check_stray_pulpcore_imports: true ci_base_image: ghcr.io/pulp/pulp-ci-centos9 ci_env: {} ci_trigger: '{pull_request: {branches: [''*'']}}' -ci_update_docs: false cli_package: pulp-cli cli_repo: https://github.com/pulp/pulp-cli.git core_import_allowed: [] @@ -20,13 +19,10 @@ deploy_client_to_pypi: true deploy_client_to_rubygems: true deploy_to_pypi: true disabled_redis_runners: [] -doc_requirements_from_pulpcore: true docker_fixtures: false -docs_test: false flake8: true flake8_ignore: [] github_org: pulp -issue_tracker: github latest_release_branch: null lint_requirements: true noissue_marker: '[noissue]' From 6eccadca35de7dbfd4436c21e2161145e4b71e16 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Wed, 26 Jun 2024 15:23:25 -0400 Subject: [PATCH 24/84] Fix upload not supporting package metadata 2.3 fixes: #682 (cherry picked from commit 781b9767dae34e8619e3831ff8ad07de1a6222ec) --- CHANGES/682.bugfix | 1 + .../functional/api/test_crud_content_unit.py | 15 +++++++++++++++ requirements.txt | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 CHANGES/682.bugfix diff --git a/CHANGES/682.bugfix b/CHANGES/682.bugfix new file mode 100644 index 00000000..4bf3e5b5 --- /dev/null +++ b/CHANGES/682.bugfix @@ -0,0 +1 @@ +Fixed uploads not supporting packages using metadata spec 2.3 diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index 2f385f3c..25834bc2 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -1,5 +1,6 @@ # coding=utf-8 """Tests that perform actions over content unit.""" +import pytest from pulp_smash.pulp3.bindings import delete_orphans, monitor_task, PulpTaskError from pulp_python.tests.functional.utils import ( @@ -12,6 +13,7 @@ from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 from tempfile import NamedTemporaryFile from urllib.parse import urljoin +from pypi_simple import PyPISimple from pulp_smash.utils import http_get from pulp_python.tests.functional.constants import ( @@ -227,3 +229,16 @@ def check_package_data(self, content_unit, expected=PYTHON_PACKAGE_DATA): for k, v in expected.items(): with self.subTest(key=k): self.assertEqual(content_unit[k], v) + + +@pytest.mark.parallel +def test_upload_metadata_23_spec(python_content_factory): + """Test that packages using metadata spec 2.3 can be uploaded to pulp.""" + filename = "urllib3-2.2.2-py3-none-any.whl" + with PyPISimple() as client: + page = client.get_project_page("urllib3") + for package in page.packages: + if package.filename == filename: + content = python_content_factory(filename, url=package.url) + assert content.metadata_version == "2.3" + break diff --git a/requirements.txt b/requirements.txt index 3d7b2059..35d01f9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ pulpcore>=3.28,<3.55 -pkginfo>=1.8.2,<1.9.7 +pkginfo>=1.10.0,<1.12.0 # Twine has <1.11 in their requirements bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 From d84ac2292166948faf736618b2fe3097fa91a97a Mon Sep 17 00:00:00 2001 From: pulpbot Date: Fri, 16 Aug 2024 12:17:44 +0000 Subject: [PATCH 25/84] Update CI files [noissue] --- .github/template_gitref | 2 +- .github/workflows/scripts/before_install.sh | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/template_gitref b/.github/template_gitref index ded489e7..5df480f7 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-377-g1ba8f46 +2021.08.26-378-g72f4b38 diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 80be9043..eaefed7d 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -57,7 +57,7 @@ fi for i in {1..3} do - ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3 + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 done if [[ $s -gt 0 ]] then diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1155aee3..7ac023e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,7 +71,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs jq jsonpatch + pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: From 957f60ce933fcc60306d3749c1201ad8beaeb8a9 Mon Sep 17 00:00:00 2001 From: Gerrod Date: Tue, 13 Aug 2024 08:33:04 -0400 Subject: [PATCH 26/84] Fix package name normalization for package pypi json view Merge pull request #717 from gerrod3/normalize-name-pypi (cherry picked from commit fc43b4b253dbec5931d6a230296942e4ba52ec3e) --- CHANGES/716.bugfix | 1 + pulp_python/app/models.py | 6 +++- pulp_python/app/pypi/views.py | 3 +- .../functional/api/test_download_content.py | 30 +++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 CHANGES/716.bugfix diff --git a/CHANGES/716.bugfix b/CHANGES/716.bugfix new file mode 100644 index 00000000..344bbf23 --- /dev/null +++ b/CHANGES/716.bugfix @@ -0,0 +1 @@ +Fixed package name normalization issue preventing syncing packages with "." or "_" in their names. diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 0e8b29fc..7f69aa9b 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -16,6 +16,7 @@ from pathlib import PurePath from .utils import ( + canonicalize_name, get_project_metadata_from_artifact, parse_project_metadata, python_content_to_json, @@ -84,6 +85,8 @@ def content_handler(self, path): ).latest("pulp_created") except ObjectDoesNotExist: return None + if len(path.parts) == 2: + path = PurePath(f"simple/{canonicalize_name(path.parts[1])}") rel_path = f"{path}/index.html" try: ca = ( @@ -100,8 +103,9 @@ def content_handler(self, path): return ArtifactResponse(ca.artifact, headers=headers) if name: + normalized = canonicalize_name(name) package_content = PythonPackageContent.objects.filter( - pk__in=self.publication.repository_version.content, name__iexact=name + pk__in=self.publication.repository_version.content, name__normalize=normalized ) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index ab276558..d2e34f6b 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -278,7 +278,8 @@ def retrieve(self, request, path, meta): elif meta_path.match("*/json"): name = meta_path.parts[0] if name: - package_content = content.filter(name__iexact=name) + normalized = canonicalize_name(name) + package_content = content.filter(name__normalize=normalized) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} json_body = python_content_to_json(path, package_content, version=version) diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 2f20cca0..236f2107 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -1,5 +1,6 @@ # coding=utf-8 """Tests that verify download of content served by Pulp.""" +import pytest import hashlib from random import choice from urllib.parse import urljoin @@ -149,3 +150,32 @@ def test_full_pulp_to_pulp_sync(self): repo3 = self._create_repo_and_sync_with_remote(remote) self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_MD_FIXTURE_SUMMARY) + + +@pytest.mark.parallel +def test_pulp2pulp_sync_with_oddities( + python_repo_with_sync, + python_remote_factory, + python_publication_factory, + python_distribution_factory, + python_content_summary, +): + """Test that Pulp can handle syncing packages with wierd names.""" + remote = python_remote_factory(includes=["oslo.utils"], url="iframe.php?url=https%3A%2F%2Fpypi.org") + repo = python_repo_with_sync(remote) + distro = python_distribution_factory(repository=repo.pulp_href) + summary = python_content_summary(repository_version=repo.latest_version_href) + # Test pulp 2 pulp full sync w/ live pypi apis + remote2 = python_remote_factory(includes=[], url=distro.base_url) + repo2 = python_repo_with_sync(remote2) + summary2 = python_content_summary(repository_version=repo2.latest_version_href) + assert summary2.present["python.python"]["count"] > 0 + assert summary.present["python.python"]["count"] == summary2.present["python.python"]["count"] + # Test w/ publication + pub = python_publication_factory(repository=repo) + distro2 = python_distribution_factory(publication=pub.pulp_href) + remote3 = python_remote_factory(includes=[], url=distro2.base_url) + repo3 = python_repo_with_sync(remote3) + summary3 = python_content_summary(repository_version=repo3.latest_version_href) + assert summary3.present["python.python"]["count"] > 0 + assert summary.present["python.python"]["count"] == summary3.present["python.python"]["count"] From 873635fbcf42b3cb7a1113fab6754d670d7d1bf5 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 21 Aug 2024 18:57:50 +0000 Subject: [PATCH 27/84] Release 3.11.3 --- .bumpversion.cfg | 2 +- CHANGES.md | 11 +++++++++++ CHANGES/682.bugfix | 1 - CHANGES/716.bugfix | 1 - docs/conf.py | 4 ++-- pulp_python/app/__init__.py | 2 +- setup.py | 2 +- 7 files changed, 16 insertions(+), 7 deletions(-) delete mode 100644 CHANGES/682.bugfix delete mode 100644 CHANGES/716.bugfix diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 06449aac..aafda73a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.11.3.dev +current_version = 3.11.3 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/CHANGES.md b/CHANGES.md index 1e7464ea..05c0de92 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,17 @@ [//]: # (towncrier release notes start) +# ## 3.11.3 (2024-08-21) {: #3.11.3 } + +#### Bugfixes {: #3.11.3-bugfix } + +- Fixed uploads not supporting packages using metadata spec 2.3 + [#682](https://github.com/pulp/pulp_python/issues/682) +- Fixed package name normalization issue preventing syncing packages with "." or "_" in their names. + [#716](https://github.com/pulp/pulp_python/issues/716) + +--- + ## 3.11.2 (2024-06-27) {: #3.11.2 } diff --git a/CHANGES/682.bugfix b/CHANGES/682.bugfix deleted file mode 100644 index 4bf3e5b5..00000000 --- a/CHANGES/682.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed uploads not supporting packages using metadata spec 2.3 diff --git a/CHANGES/716.bugfix b/CHANGES/716.bugfix deleted file mode 100644 index 344bbf23..00000000 --- a/CHANGES/716.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed package name normalization issue preventing syncing packages with "." or "_" in their names. diff --git a/docs/conf.py b/docs/conf.py index 5391b933..6a1f6bec 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.11.3.dev" +version = "3.11.3" # The full version, including alpha/beta/rc tags. -release = "3.11.3.dev" +release = "3.11.3" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 79ce0fcf..69f3bbf3 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.3.dev" + version = "3.11.3" python_package_name = "pulp-python" diff --git a/setup.py b/setup.py index 70de72c2..f91143cb 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.3.dev", + version="3.11.3", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From 389d415fadff3cb78723123d0c62153db875bdb5 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 21 Aug 2024 18:57:51 +0000 Subject: [PATCH 28/84] =?UTF-8?q?Bump=20version:=203.11.3=20=E2=86=92=203.?= =?UTF-8?q?11.4.dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulp_python/app/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index aafda73a..fe06fe6a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.11.3 +current_version = 3.11.4.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 6a1f6bec..3971ab45 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = "3.11.3" +version = "3.11.4.dev" # The full version, including alpha/beta/rc tags. -release = "3.11.3" +release = "3.11.4.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 69f3bbf3..6ec32481 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.3" + version = "3.11.4.dev" python_package_name = "pulp-python" diff --git a/setup.py b/setup.py index f91143cb..47903f0f 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.3", + version="3.11.4.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From 80a4b155a753bed3ebffe1c676e19d68e2750125 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 1 Sep 2024 02:47:06 +0000 Subject: [PATCH 29/84] Update CI files [noissue] --- .github/template_gitref | 2 +- .github/workflows/scripts/script.sh | 8 ++++---- functest_requirements.txt | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/template_gitref b/.github/template_gitref index 5df480f7..3e378489 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-378-g72f4b38 +2021.08.26-379-gece49c9 diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 60284868..0576046b 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -128,11 +128,11 @@ if [ -f "$FUNC_TEST_SCRIPT" ]; then else if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] then - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" else - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel'" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel'" fi fi pushd ../pulp-cli diff --git a/functest_requirements.txt b/functest_requirements.txt index 0beaa4d5..64118294 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -3,3 +3,5 @@ pytest<8 lxml twine pypi-simple +pytest-xdist +pytest-timeout From 4aae37db3fba63d94ee4c496db2d34ffbad44848 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Mon, 9 Sep 2024 10:36:14 +0000 Subject: [PATCH 30/84] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 3 ++- .github/template_gitref | 2 +- .github/workflows/scripts/install.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index afe0b118..360c1c6a 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -9,9 +9,10 @@ ADD ./{{ item.name }} ./{{ item.name }} # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads # Hacking botocore (https://github.com/boto/botocore/pull/1990) +# This MUST be the ONLY call to pip install in inside the container. RUN pip3 install --upgrade pip setuptools wheel && \ rm -rf /root/.cache/pip && \ - pip3 install + pip3 install pipdeptree {%- if s3_test | default(false) -%} {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} diff --git a/.github/template_gitref b/.github/template_gitref index 3e378489..adfc6a8f 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-379-gece49c9 +2021.08.26-380-g46e00a5 diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 9711e5cf..45dcdb23 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -156,5 +156,5 @@ if [[ "$TEST" = "azure" ]]; then fi echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +cmd_prefix bash -c "pip3 list && pipdeptree" echo ::endgroup:: From daa8efd3802cb680cb0975cbad78db78c0eff86c Mon Sep 17 00:00:00 2001 From: Gerrod Date: Fri, 25 Oct 2024 11:29:51 -0400 Subject: [PATCH 31/84] Merge pull request #751 from pedro-psb/fix/type-regression-on-drf-spectacular Fix Any serializer type regression breaking Ruby bindings (cherry picked from commit 4d8e8db608c80e3829b4c3ee90e84d7ef69c5236) --- CHANGES/+fix-any-type.bugfix | 2 ++ pulp_python/app/fields.py | 12 ++++++++++++ pulp_python/app/pypi/serializers.py | 7 ++++--- pulp_python/app/serializers.py | 13 +++++++------ 4 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 CHANGES/+fix-any-type.bugfix create mode 100644 pulp_python/app/fields.py diff --git a/CHANGES/+fix-any-type.bugfix b/CHANGES/+fix-any-type.bugfix new file mode 100644 index 00000000..89ea25f5 --- /dev/null +++ b/CHANGES/+fix-any-type.bugfix @@ -0,0 +1,2 @@ +Fixed the JSONField specification so it doesn't break ruby bindings. +See context [here](https://github.com/pulp/pulp_rpm/issues/3639). diff --git a/pulp_python/app/fields.py b/pulp_python/app/fields.py new file mode 100644 index 00000000..110d2c55 --- /dev/null +++ b/pulp_python/app/fields.py @@ -0,0 +1,12 @@ +from drf_spectacular.utils import extend_schema_field +from drf_spectacular.types import OpenApiTypes +from rest_framework import serializers + + +@extend_schema_field(OpenApiTypes.OBJECT) +class JSONObjectField(serializers.JSONField): + """A drf JSONField override to force openapi schema to use 'object' type. + + Not strictly correct, but we relied on that for a long time. + See: https://github.com/tfranzel/drf-spectacular/issues/1095 + """ diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index d5f6d607..906293af 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -3,6 +3,7 @@ from rest_framework import serializers from pulp_python.app.utils import DIST_EXTENSIONS +from pulp_python.app import fields from pulpcore.plugin.models import Artifact from django.db.utils import IntegrityError @@ -27,9 +28,9 @@ class PackageMetadataSerializer(serializers.Serializer): """ last_serial = serializers.IntegerField(help_text=_("Cache value from last PyPI sync")) - info = serializers.JSONField(help_text=_("Core metadata of the package")) - releases = serializers.JSONField(help_text=_("List of all the releases of the package")) - urls = serializers.JSONField() + info = fields.JSONObjectField(help_text=_("Core metadata of the package")) + releases = fields.JSONObjectField(help_text=_("List of all the releases of the package")) + urls = fields.JSONObjectField() class PackageUploadSerializer(serializers.Serializer): diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 2573a1e1..36fb6395 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -7,6 +7,7 @@ from pulpcore.plugin import serializers as core_serializers from pulp_python.app import models as python_models +from pulp_python.app import fields from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata @@ -154,7 +155,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_('A browsable URL for the project and a label for it, separated by a comma.') ) - project_urls = serializers.JSONField( + project_urls = fields.JSONObjectField( required=False, default=dict, help_text=_('A dictionary of labels and URLs for the project.') ) @@ -167,28 +168,28 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ') ) - requires_dist = serializers.JSONField( + requires_dist = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing names of some other distutils project ' 'required by this distribution.') ) - provides_dist = serializers.JSONField( + provides_dist = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing names of a Distutils project which is contained' ' within this distribution.') ) - obsoletes_dist = serializers.JSONField( + obsoletes_dist = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing names of a distutils project\'s distribution which ' 'this distribution renders obsolete, meaning that the two projects should not ' 'be installed at the same time.') ) - requires_external = serializers.JSONField( + requires_external = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing some dependency in the system that the distribution ' 'is to be used.') ) - classifiers = serializers.JSONField( + classifiers = fields.JSONObjectField( required=False, default=list, help_text=_('A JSON list containing classification values for a Python package.') ) From 7427e6cbd95bdb9c9354411197c485667a1e3e75 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 3 Nov 2024 02:48:29 +0000 Subject: [PATCH 32/84] Update CI files --- .ci/ansible/Containerfile.j2 | 2 +- .ci/scripts/check_release.py | 11 +++-- .ci/scripts/collect_changes.py | 2 +- .ci/scripts/pr_labels.py | 60 ++++++++++++++++++++++++++ .ci/scripts/validate_commit_message.py | 11 ----- .github/template_gitref | 2 +- .github/workflows/create-branch.yml | 2 - .github/workflows/pr_checks.yml | 55 ++++++++++++----------- .github/workflows/scripts/install.sh | 2 +- .github/workflows/scripts/script.sh | 10 +++-- .github/workflows/test.yml | 2 +- .github/workflows/update_ci.yml | 6 +-- MANIFEST.in | 1 + releasing.md | 27 ++++++++++++ template_config.yml | 3 +- 15 files changed, 139 insertions(+), 57 deletions(-) create mode 100755 .ci/scripts/pr_labels.py create mode 100644 releasing.md diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 360c1c6a..d57b44ed 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -12,7 +12,7 @@ ADD ./{{ item.name }} ./{{ item.name }} # This MUST be the ONLY call to pip install in inside the container. RUN pip3 install --upgrade pip setuptools wheel && \ rm -rf /root/.cache/pip && \ - pip3 install pipdeptree + pip3 install {%- if s3_test | default(false) -%} {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 4f2f3172..a68c7715 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -64,6 +64,7 @@ def main(): for branch in branches: if branch != DEFAULT_BRANCH: # Check if a Z release is needed + reasons = [] changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") z_changelog = False for change in changes.split("\n"): @@ -76,23 +77,27 @@ def main(): ) elif ext in Z_CHANGELOG_EXTS: z_changelog = True + if z_changelog: + reasons.append("Backports") last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}") req_txt_diff = repo.git.diff( f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" ) - if z_changelog or req_txt_diff: + if req_txt_diff: + reasons.append("requirements.txt") + + if reasons: curr_version = Version(last_tag) assert curr_version.base_version.startswith( branch ), "Current-version has to belong to the current branch!" next_version = Version(f"{branch}.{curr_version.micro + 1}") - reason = "CHANGES" if z_changelog else "requirements.txt" print( f"A Z-release is needed for {branch}, " f"Prev: {last_tag}, " f"Next: {next_version.base_version}, " - f"Reason: {reason}" + f"Reason: {','.join(reasons)}" ) releases.append(next_version) else: diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index b6c64eb4..8cae45a9 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -103,7 +103,7 @@ def main(): for change in main_changes: fp.write(change[1]) - repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE) + repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE) if __name__ == "__main__": diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py new file mode 100755 index 00000000..0c478a21 --- /dev/null +++ b/.ci/scripts/pr_labels.py @@ -0,0 +1,60 @@ +#!/bin/env python3 + +# This script is running with elevated privileges from the main branch against pull requests. + +import re +import sys +import tomllib +from pathlib import Path + +from git import Repo + + +def main(): + assert len(sys.argv) == 3 + + with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) + BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT") + ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)") + CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$") + try: + CHANGELOG_EXTS = { + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + } + except KeyError: + CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"} + + repo = Repo(".") + + base_commit = repo.commit(sys.argv[1]) + head_commit = repo.commit(sys.argv[2]) + + pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}")) + + labels = { + "multi-commit": len(pr_commits) > 1, + "cherry-pick": False, + "no-issue": False, + "no-changelog": False, + "wip": False, + } + for commit in pr_commits: + labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None + no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None + labels["no-issue"] |= no_issue + cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None + labels["cherry-pick"] |= cherry_pick + changelog_snippets = [ + k + for k in commit.stats.files + if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS + ] + labels["no-changelog"] |= not changelog_snippets + + print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v))) + print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v))) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index 9c718a83..739f75c5 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -13,7 +13,6 @@ import warnings from github import Github -NO_ISSUE = "[noissue]" CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] KEYWORDS = ["fixes", "closes"] @@ -60,15 +59,5 @@ def __check_changelog(issue): for issue in pattern.findall(message): __check_status(issue) __check_changelog(issue) -else: - if NO_ISSUE in message: - print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) - elif "Merge" in message and "cherry picked from commit" in message: - pass - else: - sys.exit( - "Error: no attached issues found for {sha}. If this was intentional, add " - " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) - ) print("Commit message for {sha} passed.".format(sha=sha[0:7])) diff --git a/.github/template_gitref b/.github/template_gitref index adfc6a8f..77b8478b 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-380-g46e00a5 +2021.08.26-388-g624de1a diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index b35c4bef..3c3bda68 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -93,10 +93,8 @@ jobs: branch: minor-version-bump base: main title: Bump minor version - body: '[noissue]' commit-message: | Bump minor version - [noissue] delete-branch: true - name: Push release branch diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index f2a44ab6..c4898097 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -6,57 +6,62 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Python PR static checks +name: "Python PR static checks" on: pull_request_target: - types: [opened, synchronize, reopened] + types: ["opened", "synchronize", "reopened"] # This workflow runs with elevated permissions. # Do not even think about running a single bit of code from the PR. # Static analysis should be fine however. concurrency: - group: ${{ github.event.pull_request.number }}-${{ github.workflow }} + group: "${{ github.event.pull_request.number }}-${{ github.workflow }}" cancel-in-progress: true jobs: - single_commit: - runs-on: ubuntu-latest - name: Label multiple commit PR + apply_labels: + runs-on: "ubuntu-latest" + name: "Label PR" permissions: - pull-requests: write + pull-requests: "write" steps: - uses: "actions/checkout@v4" with: fetch-depth: 0 - - name: Commit Count Check + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Determine PR labels" run: | + pip install GitPython==3.1.42 git fetch origin ${{ github.event.pull_request.head.sha }} - echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV" - - uses: actions/github-script@v7 + python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" + - uses: "actions/github-script@v7" + name: "Apply PR Labels" with: script: | - const labelName = "multi-commit"; - const { COMMIT_COUNT } = process.env; + const { ADD_LABELS, REMOVE_LABELS } = process.env; - if (COMMIT_COUNT == 1) - { - try { - await github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - }); - } catch(err) { + if (REMOVE_LABELS.length) { + for await (const labelName of REMOVE_LABELS.split(",")) { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } } } - else - { + if (ADD_LABELS.length) { await github.rest.issues.addLabels({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - labels: [labelName], + labels: ADD_LABELS.split(","), }); } +... diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 45dcdb23..b5a927b6 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -156,5 +156,5 @@ if [[ "$TEST" = "azure" ]]; then fi echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pipdeptree" +cmd_prefix bash -c "pip3 list" echo ::endgroup:: diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 0576046b..8e60f87d 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -46,12 +46,14 @@ pushd ../pulp-openapi-generator # Workaround: Domains are not supported by the published bindings. # Sadly: Different pulpcore-versions aren't either... - # So we exclude the prebuilt ones only for domains disabled. - if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ] + # * In the 'pulp' scenario we use the published/prebuilt bindings, so we can test it. + # * In other scenarios we generate new bindings from server spec, so we have a more + # reliable client. + if [ "$TEST" = "pulp" ] then - BUILT_CLIENTS="" - else BUILT_CLIENTS=" python " + else + BUILT_CLIENTS="" fi for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ac023e9..414aec0e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,5 +147,5 @@ jobs: docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + docker exec pulp bash -c "pip3 list" || true ... diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index e51899f4..02f251fa 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -62,11 +62,7 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch main" - body: "" branch: "update-ci/main" base: "main" - commit-message: | - Update CI files - - [noissue] delete-branch: true +... diff --git a/MANIFEST.in b/MANIFEST.in index f9759e6f..e519d8a0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,3 +8,4 @@ include functest_requirements.txt include test_requirements.txt include unittest_requirements.txt include pulp_python/app/webserver_snippets/* +exclude releasing.md diff --git a/releasing.md b/releasing.md new file mode 100644 index 00000000..1326954e --- /dev/null +++ b/releasing.md @@ -0,0 +1,27 @@ +[//]: # "WARNING: DO NOT EDIT!" +[//]: # "" +[//]: # "This file was generated by plugin_template, and is managed by it. Please use" +[//]: # "'./plugin-template --github pulp_python' to update this file." +[//]: # "" +[//]: # "For more info visit https://github.com/pulp/plugin_template" +# Releasing (For Internal Use) + +This document outlines the steps to perform a release. + +### Determine if a Release is Required +- Run the release checker script: + ``` + python3 .ci/scripts/check_release.py + ``` + +### Create a New Y-branch (e.g., 3.23) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_python/actions/workflows/create-branch.yml) job via the GitHub Actions. +- Look for the "Bump minor version" pull request and merge it. + +### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_python/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. + +### Final Steps (Optional but Recommended) +- Ensure the new version appears on PyPI. +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request. +- Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). diff --git a/template_config.yml b/template_config.yml index e7b07822..69d8a175 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-377-g1ba8f46 +# generated with plugin_template@2021.08.26-388-g624de1a api_root: /pulp/ black: false @@ -25,7 +25,6 @@ flake8_ignore: [] github_org: pulp latest_release_branch: null lint_requirements: true -noissue_marker: '[noissue]' os_required_packages: [] parallel_test_workers: 8 plugin_app_label: python From 78a5b740073237a19a4f080d54b6294a3a6a9920 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 10 Nov 2024 02:46:07 +0000 Subject: [PATCH 33/84] Update CI files --- .github/template_gitref | 2 +- releasing.md | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/template_gitref b/.github/template_gitref index 77b8478b..73f0df79 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-388-g624de1a +2021.08.26-389-g444ab6c diff --git a/releasing.md b/releasing.md index 1326954e..db7470b2 100644 --- a/releasing.md +++ b/releasing.md @@ -9,19 +9,21 @@ This document outlines the steps to perform a release. ### Determine if a Release is Required +- Make sure to have GitPython python package installed - Run the release checker script: ``` python3 .ci/scripts/check_release.py ``` -### Create a New Y-branch (e.g., 3.23) -- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_python/actions/workflows/create-branch.yml) job via the GitHub Actions. +### Release a New Y-version (e.g., 3.23.0) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_python/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. - Look for the "Bump minor version" pull request and merge it. +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_python/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. ### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) - Trigger a [Release Pipeline](https://github.com/pulp/pulp_python/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. -### Final Steps (Optional but Recommended) -- Ensure the new version appears on PyPI. -- Verify that the changelog has been updated by looking for the "Update Changelog" pull request. -- Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). +## Final Steps +- Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulp_python/actions/workflows/publish.yml) workflow succeeds). +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). +- [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). From 9167e5ad5b835220e7b9ef3e256a498fc81f0cef Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 17 Nov 2024 02:52:09 +0000 Subject: [PATCH 34/84] Update CI files --- .ci/assets/release_requirements.txt | 2 +- .ci/scripts/calc_constraints.py | 47 ++++++++++++++------ .ci/scripts/check_release.py | 7 +++ .ci/scripts/check_requirements.py | 49 +++++++++++++++------ .github/template_gitref | 2 +- .github/workflows/build.yml | 4 +- .github/workflows/ci.yml | 3 -- .github/workflows/create-branch.yml | 6 +-- .github/workflows/lint.yml | 39 ++++++++++------ .github/workflows/release.yml | 2 +- .github/workflows/scripts/before_install.sh | 5 --- .github/workflows/scripts/release.sh | 7 +-- lint_requirements.txt | 3 +- 13 files changed, 115 insertions(+), 61 deletions(-) diff --git a/.ci/assets/release_requirements.txt b/.ci/assets/release_requirements.txt index c064e947..6635a872 100644 --- a/.ci/assets/release_requirements.txt +++ b/.ci/assets/release_requirements.txt @@ -1,3 +1,3 @@ -bump2version +bump-my-version gitpython towncrier diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index badca1c9..35315153 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -13,6 +13,11 @@ from packaging.version import Version import yaml +try: + import tomllib +except ImportError: + import tomli as tomllib + CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" @@ -59,11 +64,11 @@ def to_upper_bound(req): operator = "~=" version = Version(spec.version) if version.micro != 0: - max_version = f"{version.major}.{version.minor}.{version.micro-1}" + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" elif version.minor != 0: - max_version = f"{version.major}.{version.minor-1}" + max_version = f"{version.major}.{version.minor - 1}" elif version.major != 0: - max_version = f"{version.major-1}.0" + max_version = f"{version.major - 1}.0" else: return f"# NO BETTER CONSTRAINT: {req}" return f"{requirement.name}{operator}{max_version}" @@ -100,18 +105,32 @@ def main(): parser.add_argument("filename", nargs="*") args = parser.parse_args() - with fileinput.input(files=args.filename) as req_file: - for line in req_file: - if line.strip().startswith("#"): - # Shortcut comment only lines - print(line.strip()) - else: - req, comment = split_comment(line) - if args.upper: - new_req = to_upper_bound(req) + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) else: - new_req = to_lower_bound(req) - print(new_req + comment) + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) if __name__ == "__main__": diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index a68c7715..a70abc21 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -86,6 +86,13 @@ def main(): ) if req_txt_diff: reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.append( + "pyproject.toml changed (PLEASE check if dependencies are affected." + ) if reasons: curr_version = Version(last_tag) diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index 671bade1..cf9efbe9 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,11 +5,13 @@ # # For more info visit https://github.com/pulp/plugin_template +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -20,17 +22,33 @@ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -49,18 +67,21 @@ and req.name != "pulp-python-client" ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") - ops = [op for op, ver in req.specs] - spec = str(req.specs) + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) +if __name__ == "__main__": + main() diff --git a/.github/template_gitref b/.github/template_gitref index 73f0df79..d37a43c8 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-389-g444ab6c +2021.08.26-393-g0e700c1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d1fcf2e5..67878dc4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,11 +34,11 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel mkdocs jq + pip install build packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Build package" run: | - python3 setup.py sdist bdist_wheel --python-tag py3 + python3 -m build twine check dist/* - name: "Install built packages" run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db87bfcc..74f73b58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,9 +42,6 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" run: | .github/workflows/scripts/check_commit.sh - - name: "Verify requirements files" - run: | - python .ci/scripts/check_requirements.py docs: uses: "./.github/workflows/docs.yml" diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 3c3bda68..9ea30712 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -39,7 +39,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version packaging -r plugin_template/requirements.txt + pip install bump-my-version packaging -r plugin_template/requirements.txt echo ::endgroup:: - name: "Setting secrets" @@ -54,7 +54,7 @@ jobs: run: | # Just to be sure... git checkout main - NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')" + NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')" if [ -z "$NEW_BRANCH" ] then echo Could not determine the new branch name. @@ -70,7 +70,7 @@ jobs: - name: Bump version on main branch working-directory: pulp_python run: | - bump2version --no-commit minor + bump-my-version bump --no-commit minor - name: Remove entries from CHANGES directory working-directory: pulp_python diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 701d990e..9dddcbdd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ defaults: jobs: lint: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v4" @@ -34,23 +34,36 @@ jobs: pip install -r lint_requirements.txt echo ::endgroup:: - - name: Lint workflow files + - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + # Lint code. - - name: Run flake8 - run: flake8 + - name: "Run flake8" + run: | + flake8 - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py + + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c362a0a3..b758bae6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version towncrier + pip install bump-my-version towncrier echo ::endgroup:: - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index eaefed7d..c31ec734 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -45,11 +45,6 @@ if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 -fi - if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh index 9525f229..a08353cd 100755 --- a/.github/workflows/scripts/release.sh +++ b/.github/workflows/scripts/release.sh @@ -10,7 +10,8 @@ then exit 1 fi -NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +# The tail is a necessary workaround to remove the warning from the output. +NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)" echo "Release ${NEW_VERSION}" if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] @@ -20,7 +21,7 @@ then fi towncrier build --yes --version "${NEW_VERSION}" -bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty -bump2version patch --commit +bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump-my-version bump patch --commit git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/lint_requirements.txt b/lint_requirements.txt index 2422f937..a2936209 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,7 +5,8 @@ # # For more info visit https://github.com/pulp/plugin_template -# python packages handy for developers, but not required by pulp +bump-my-version check-manifest flake8 +packaging yamllint From d8e361b977db7229f2972818bb20743f7cd3d454 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 24 Nov 2024 02:53:48 +0000 Subject: [PATCH 35/84] Update CI files --- .bumpversion.cfg | 21 -------- .ci/scripts/check_release.py | 76 +++++++++++++++++++--------- .ci/scripts/check_requirements.py | 5 +- .github/template_gitref | 2 +- .github/workflows/scripts/install.sh | 2 +- .github/workflows/test.yml | 2 +- pyproject.toml | 31 ++++++++++++ 7 files changed, 86 insertions(+), 53 deletions(-) delete mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index fe06fe6a..00000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[bumpversion] -current_version = 3.11.4.dev -commit = False -tag = False -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? -serialize = - {major}.{minor}.{patch}.{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = prod -first_value = dev -values = - dev - prod - -[bumpversion:file:./pulp_python/app/__init__.py] - -[bumpversion:file:./setup.py] - -[bumpversion:file:./docs/conf.py] diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index a70abc21..095dd739 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -1,28 +1,21 @@ #!/usr/bin/env python -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - import argparse import re import os +import tomllib import yaml +from pathlib import Path from tempfile import TemporaryDirectory from packaging.version import Version from git import Repo -UPSTREAM_REMOTE = "https://github.com/pulp/pulp_python.git" -DEFAULT_BRANCH = "main" RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"] Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"] -def main(): +def options(): """Check which branches need a release.""" parser = argparse.ArgumentParser() parser.add_argument( @@ -32,17 +25,57 @@ def main(): "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) - opts = parser.parse_args() + return parser.parse_args() + + +def template_config(): + # Assume this script lies in .ci/scripts + path = Path(__file__).absolute().parent.parent.parent / "template_config.yml" + return yaml.safe_load(path.read_text()) + +def current_version(repo, commitish): + try: + pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml")) + current_version = pyproject_toml["project"]["version"] + except Exception: + current_version = repo.git.grep( + "current_version", commitish, "--", ".bumpversion.cfg" + ).split("=")[-1] + return Version(current_version) + + +def check_pyproject_dependencies(repo, from_commit, to_commit): + try: + old_pyproject = tomllib.load(repo.show("{from_commit}:pyproject.toml")) + old_dependencies = set(old_pyproject["project"]["dependencies"]) + new_pyproject = tomllib.load(repo.show("{to_commit}:pyproject.toml")) + new_dependencies = set(new_pyproject["project"]["dependencies"]) + if old_dependencies != new_dependencies: + return ["dependencies"] + else: + return [] + except Exception as e: + print(f"WARNING: Comparing the dependencies in pyproject.toml failed. ({e})") + # Gathering more details failed. + return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."] + + +def main(options, template_config): with TemporaryDirectory() as d: # Clone from upstream to ensure we have updated branches & main + GITHUB_ORG = template_config["github_org"] + PLUGIN_NAME = template_config["plugin_name"] + UPSTREAM_REMOTE = f"https://github.com/{GITHUB_ORG}/{PLUGIN_NAME}.git" + DEFAULT_BRANCH = template_config["plugin_default_branch"] + repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none") heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")] available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] available_branches.sort(key=lambda ver: Version(ver)) available_branches.append(DEFAULT_BRANCH) - branches = opts.branches + branches = options.branches if branches == "supported": with open(f"{d}/template_config.yml", mode="r") as f: tc = yaml.safe_load(f) @@ -90,9 +123,7 @@ def main(): f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" ) if pyproject_diff: - reasons.append( - "pyproject.toml changed (PLEASE check if dependencies are affected." - ) + reasons.extend(check_pyproject_dependencies(repo, last_tag, f"origin/{branch}")) if reasons: curr_version = Version(last_tag) @@ -113,15 +144,10 @@ def main(): for change in changes.split("\n"): _, ext = os.path.splitext(change) if ext in Y_CHANGELOG_EXTS: - # We don't put Y release bumps in the commit message, check file instead - # The 'current_version' is always the next version to release - next_version = repo.git.grep( - "current_version", DEFAULT_BRANCH, "--", ".bumpversion.cfg" - ).split("=")[-1] - next_version = Version(next_version) - print( - f"A new Y-release is needed! New Version: {next_version.base_version}" - ) + # We don't put Y release bumps in the commit message, check file instead. + # The 'current_version' is always the dev of the next version to release. + next_version = current_version(repo, DEFAULT_BRANCH).base_version + print(f"A new Y-release is needed! New Version: {next_version}") releases.append(next_version) break @@ -130,4 +156,4 @@ def main(): if __name__ == "__main__": - main() + main(options(), template_config()) diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index cf9efbe9..ec871a10 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -62,10 +62,7 @@ def main(): else: if check_prereleases and req.specifier.prereleases: # Do not even think about begging for more exceptions! - if ( - not req.name.startswith("opentelemetry") - and req.name != "pulp-python-client" - ): + if req.name != "pulp-python-client": errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") ops = [spec.operator for spec in req.specifier] if "~=" in ops: diff --git a/.github/template_gitref b/.github/template_gitref index d37a43c8..8156a7ac 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-393-g0e700c1 +2021.08.26-399-g78ad960 diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index b5a927b6..e0bf5b20 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,7 +15,7 @@ set -euv source .github/workflows/scripts/utils.sh -PLUGIN_VERSION="$(sed -n -e 's/^\s*current_version\s*=\s*//p' .bumpversion.cfg | python -c 'from packaging.version import Version; print(Version(input()))')" +PLUGIN_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" PLUGIN_SOURCE="./pulp_python/dist/pulp_python-${PLUGIN_VERSION}-py3-none-any.whl" export PULP_API_ROOT="/pulp/" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 414aec0e..f2271930 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,7 +71,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch + pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: diff --git a/pyproject.toml b/pyproject.toml index 6f2c221a..f9a65cce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,3 +27,34 @@ ignore = [ "lint_requirements.txt", ".flake8", ] + +[tool.bumpversion] +# This section is managed by the plugin template. Do not edit manually. + +current_version = "3.11.4.dev" +commit = false +tag = false +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(\\.(?P[a-z]+))?" +serialize = [ + "{major}.{minor}.{patch}.{release}", + "{major}.{minor}.{patch}", +] + +[tool.bumpversion.parts.release] +# This section is managed by the plugin template. Do not edit manually. + +optional_value = "prod" +values = [ + "dev", + "prod", +] + +[[tool.bumpversion.files]] +# This section is managed by the plugin template. Do not edit manually. + +filename = "./pulp_python/app/__init__.py" +search = "version = \"{current_version}\"" +replace = "version = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "./setup.py" \ No newline at end of file From 0f1abe7a50ae2036e28ab19cbdb887102b82e5cd Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 1 Dec 2024 03:00:45 +0000 Subject: [PATCH 36/84] Update CI files --- .ci/scripts/check_release.py | 5 ++++- .ci/scripts/check_requirements.py | 5 ++++- .github/template_gitref | 2 +- .github/workflows/ci.yml | 3 +++ .github/workflows/nightly.yml | 4 ++++ .github/workflows/scripts/publish_plugin_pypi.sh | 4 ++-- .github/workflows/test.yml | 14 +++++--------- 7 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 095dd739..da45be6a 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -37,7 +37,10 @@ def template_config(): def current_version(repo, commitish): try: pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml")) - current_version = pyproject_toml["project"]["version"] + try: + current_version = pyproject_toml["project"]["version"] + except Exception: + current_version = pyproject_toml["tool"]["bumpversion"]["current_version"] except Exception: current_version = repo.git.grep( "current_version", commitish, "--", ".bumpversion.cfg" diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index ec871a10..cf9efbe9 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -62,7 +62,10 @@ def main(): else: if check_prereleases and req.specifier.prereleases: # Do not even think about begging for more exceptions! - if req.name != "pulp-python-client": + if ( + not req.name.startswith("opentelemetry") + and req.name != "pulp-python-client" + ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") ops = [spec.operator for spec in req.specifier] if "~=" in ops: diff --git a/.github/template_gitref b/.github/template_gitref index 8156a7ac..70eef899 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-399-g78ad960 +2021.08.26-405-g3845bbc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74f73b58..d82f9fee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,9 @@ jobs: test: needs: "build" uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] deprecations: runs-on: "ubuntu-latest" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index eeb6ae45..f155d81c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -29,6 +29,9 @@ jobs: test: needs: "build" uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] changelog: runs-on: ubuntu-latest @@ -65,3 +68,4 @@ jobs: branch: "changelog/update" delete-branch: true path: "pulp_python" +... diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index 1dd7d883..384cecb2 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -28,6 +28,6 @@ then fi twine upload -u __token__ -p "$PYPI_API_TOKEN" \ -"dist/pulp_python-$VERSION-py3-none-any.whl" \ -"dist/pulp-python-$VERSION.tar.gz" \ +dist/pulp?python-"$VERSION"-py3-none-any.whl \ +dist/pulp?python-"$VERSION".tar.gz \ ; diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f2271930..a18b9230 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,10 @@ name: "Test" on: workflow_call: + inputs: + matrix_env: + required: true + type: string defaults: run: @@ -20,11 +24,7 @@ jobs: strategy: fail-fast: false matrix: - env: - - TEST: pulp - - TEST: azure - - TEST: s3 - - TEST: lowerbounds + env: ${{ fromJSON(inputs.matrix_env) }} steps: - uses: "actions/checkout@v4" @@ -88,10 +88,6 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - uses: ruby/setup-ruby@v1 - if: ${{ env.TEST == 'pulp' }} - with: - ruby-version: "2.6" - name: "Install" run: | From ef4338a39224e1f427e135128214dae609379007 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 8 Dec 2024 02:57:54 +0000 Subject: [PATCH 37/84] Update CI files --- .github/template_gitref | 2 +- pyproject.toml | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/template_gitref b/.github/template_gitref index 70eef899..22e8d0a9 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-405-g3845bbc +2021.08.26-406-g5f397e3 diff --git a/pyproject.toml b/pyproject.toml index f9a65cce..c6174984 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,19 +34,33 @@ ignore = [ current_version = "3.11.4.dev" commit = false tag = false -parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(\\.(?P[a-z]+))?" +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" serialize = [ - "{major}.{minor}.{patch}.{release}", - "{major}.{minor}.{patch}", + "{major}.{minor}.{patch}.{release}", + "{major}.{minor}.{patch}", + "{major}.{minor}.{alpha}{patch}.{release}", + "{major}.{minor}.{alpha}{patch}", ] +[tool.bumpversion.parts.alpha] +# This section is managed by the plugin template. Do not edit manually. + +# This is sort of a hack. In PEP440 prerelease markers work quite differently. +# But this fits best with the way we have been doing release versions. +optional_value = "final" +values = [ + "0a", + "final", +] +independent = true + [tool.bumpversion.parts.release] # This section is managed by the plugin template. Do not edit manually. optional_value = "prod" values = [ - "dev", - "prod", + "dev", + "prod", ] [[tool.bumpversion.files]] From 49edcf013a286078756f35aa1b54b6d0d981fbf9 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Mon, 16 Dec 2024 10:41:04 +0000 Subject: [PATCH 38/84] Update CI files --- .ci/scripts/check_release.py | 197 ++++++++++-------- .ci/scripts/validate_commit_message.py | 59 ++++-- .github/template_gitref | 2 +- .github/workflows/pr_checks.yml | 8 +- .github/workflows/scripts/check_commit.sh | 4 - .../workflows/scripts/publish_client_gem.sh | 13 +- .../workflows/scripts/publish_client_pypi.sh | 17 +- .../workflows/scripts/publish_plugin_pypi.sh | 17 +- .github/workflows/update_ci.yml | 9 + 9 files changed, 175 insertions(+), 151 deletions(-) diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index da45be6a..dfc4a41e 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -6,7 +6,6 @@ import tomllib import yaml from pathlib import Path -from tempfile import TemporaryDirectory from packaging.version import Version from git import Repo @@ -25,6 +24,12 @@ def options(): "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) + parser.add_argument( + "--no-fetch", + default=False, + action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true", + help="Don't fetch remote. Run faster at the expense of maybe being outdated.", + ) return parser.parse_args() @@ -50,10 +55,15 @@ def current_version(repo, commitish): def check_pyproject_dependencies(repo, from_commit, to_commit): try: - old_pyproject = tomllib.load(repo.show("{from_commit}:pyproject.toml")) + new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml")) + try: + new_dependencies = set(new_pyproject["project"]["dependencies"]) + except KeyError: + # New branch does not declare dependencies in pyproject.toml. + # Assume no release needed for this reason. + return [] + old_pyproject = tomllib.loads(repo.git.show(f"{from_commit}:pyproject.toml")) old_dependencies = set(old_pyproject["project"]["dependencies"]) - new_pyproject = tomllib.load(repo.show("{to_commit}:pyproject.toml")) - new_dependencies = set(new_pyproject["project"]["dependencies"]) if old_dependencies != new_dependencies: return ["dependencies"] else: @@ -65,97 +75,100 @@ def check_pyproject_dependencies(repo, from_commit, to_commit): def main(options, template_config): - with TemporaryDirectory() as d: - # Clone from upstream to ensure we have updated branches & main - GITHUB_ORG = template_config["github_org"] - PLUGIN_NAME = template_config["plugin_name"] - UPSTREAM_REMOTE = f"https://github.com/{GITHUB_ORG}/{PLUGIN_NAME}.git" - DEFAULT_BRANCH = template_config["plugin_default_branch"] - - repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none") - heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")] - available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] - available_branches.sort(key=lambda ver: Version(ver)) - available_branches.append(DEFAULT_BRANCH) - - branches = options.branches - if branches == "supported": - with open(f"{d}/template_config.yml", mode="r") as f: - tc = yaml.safe_load(f) - branches = set(tc["supported_release_branches"]) - latest_release_branch = tc["latest_release_branch"] - if latest_release_branch is not None: - branches.add(latest_release_branch) - branches.add(DEFAULT_BRANCH) - else: - branches = set(branches.split(",")) - - if diff := branches - set(available_branches): - print(f"Supplied branches contains non-existent branches! {diff}") - exit(1) - - print(f"Checking for releases on branches: {branches}") - - releases = [] - for branch in branches: - if branch != DEFAULT_BRANCH: - # Check if a Z release is needed - reasons = [] - changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") - z_changelog = False - for change in changes.split("\n"): - # Check each changelog file to make sure everything checks out - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - print( - f"Warning: A non-backported changelog ({change}) is present in the " - f"{branch} release branch!" - ) - elif ext in Z_CHANGELOG_EXTS: - z_changelog = True - if z_changelog: - reasons.append("Backports") - - last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}") - req_txt_diff = repo.git.diff( - f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" - ) - if req_txt_diff: - reasons.append("requirements.txt") - pyproject_diff = repo.git.diff( - f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" - ) - if pyproject_diff: - reasons.extend(check_pyproject_dependencies(repo, last_tag, f"origin/{branch}")) - - if reasons: - curr_version = Version(last_tag) - assert curr_version.base_version.startswith( - branch - ), "Current-version has to belong to the current branch!" - next_version = Version(f"{branch}.{curr_version.micro + 1}") + DEFAULT_BRANCH = template_config["plugin_default_branch"] + + repo = Repo() + + upstream_default_branch = next( + (branch for branch in repo.branches if branch.name == DEFAULT_BRANCH) + ).tracking_branch() + remote = upstream_default_branch.remote_name + if not options.no_fetch: + repo.remote(remote).fetch() + + # Warning: This will not work if branch names contain "/" but we don't really care here. + heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")] + available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] + available_branches.sort(key=lambda ver: Version(ver)) + available_branches.append(DEFAULT_BRANCH) + + branches = options.branches + if branches == "supported": + tc = yaml.safe_load(repo.git.show(f"{upstream_default_branch}:template_config.yml")) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) + else: + branches = set(branches.split(",")) + + if diff := branches - set(available_branches): + print(f"Supplied branches contains non-existent branches! {diff}") + exit(1) + + print(f"Checking for releases on branches: {branches}") + + releases = [] + for branch in branches: + if branch != DEFAULT_BRANCH: + # Check if a Z release is needed + reasons = [] + changes = repo.git.ls_tree("-r", "--name-only", f"{remote}/{branch}", "CHANGES/") + z_changelog = False + for change in changes.split("\n"): + # Check each changelog file to make sure everything checks out + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: print( - f"A Z-release is needed for {branch}, " - f"Prev: {last_tag}, " - f"Next: {next_version.base_version}, " - f"Reason: {','.join(reasons)}" + f"Warning: A non-backported changelog ({change}) is present in the " + f"{branch} release branch!" ) + elif ext in Z_CHANGELOG_EXTS: + z_changelog = True + if z_changelog: + reasons.append("Backports") + + last_tag = repo.git.describe("--tags", "--abbrev=0", f"{remote}/{branch}") + req_txt_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "requirements.txt" + ) + if req_txt_diff: + reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.extend(check_pyproject_dependencies(repo, last_tag, f"{remote}/{branch}")) + + if reasons: + curr_version = Version(last_tag) + assert curr_version.base_version.startswith( + branch + ), "Current-version has to belong to the current branch!" + next_version = Version(f"{branch}.{curr_version.micro + 1}") + print( + f"A Z-release is needed for {branch}, " + f"Prev: {last_tag}, " + f"Next: {next_version.base_version}, " + f"Reason: {','.join(reasons)}" + ) + releases.append(next_version) + else: + # Check if a Y release is needed + changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") + for change in changes.split("\n"): + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: + # We don't put Y release bumps in the commit message, check file instead. + # The 'current_version' is always the dev of the next version to release. + next_version = current_version(repo, DEFAULT_BRANCH).base_version + print(f"A new Y-release is needed! New Version: {next_version}") releases.append(next_version) - else: - # Check if a Y release is needed - changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") - for change in changes.split("\n"): - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - # We don't put Y release bumps in the commit message, check file instead. - # The 'current_version' is always the dev of the next version to release. - next_version = current_version(repo, DEFAULT_BRANCH).base_version - print(f"A new Y-release is needed! New Version: {next_version}") - releases.append(next_version) - break - - if len(releases) == 0: - print("No new releases to perform.") + break + + if len(releases) == 0: + print("No new releases to perform.") if __name__ == "__main__": diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index 739f75c5..04e3fd30 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -5,37 +5,57 @@ # # For more info visit https://github.com/pulp/plugin_template +import os import re +import subprocess import sys +import tomllib from pathlib import Path -import subprocess -import os -import warnings + from github import Github -CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] +with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) KEYWORDS = ["fixes", "closes"] +BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"Apply suggestions from code review", +] +try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] +except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] +NOISSUE_MARKER = "[noissue]" sha = sys.argv[1] message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") +if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + +if any((re.match(pattern, message, re.IGNORECASE) for pattern in BLOCKING_REGEX)): + sys.exit("This PR is not ready for consumption.") + g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp_python") -def __check_status(issue): +def check_status(issue): gi = repo.get_issue(int(issue)) if gi.pull_request: sys.exit(f"Error: issue #{issue} is a pull request.") - if gi.closed_at and "cherry picked from commit" not in message: - warnings.warn( - "When backporting, use the -x flag to append a line that says " - "'(cherry picked from commit ...)' to the original commit message." - ) + if gi.closed_at: sys.exit(f"Error: issue #{issue} is closed.") -def __check_changelog(issue): +def check_changelog(issue): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -43,21 +63,20 @@ def __check_changelog(issue): for match in matches: if match.suffix not in CHANGELOG_EXTS: sys.exit(f"Invalid extension for changelog entry '{match}'.") - if match.suffix == ".feature" and "cherry picked from commit" in message: - sys.exit(f"Can not backport '{match}' as it is a feature.") print("Checking commit message for {sha}.".format(sha=sha[0:7])) # validate the issue attached to the commit -regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -pattern = re.compile(regex, re.IGNORECASE) - -issues = pattern.findall(message) +issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) +issues = re.findall(issue_regex, message, re.IGNORECASE) +cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" +cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) if issues: - for issue in pattern.findall(message): - __check_status(issue) - __check_changelog(issue) + for issue in issues: + if not cherry_pick: + check_status(issue) + check_changelog(issue) print("Commit message for {sha} passed.".format(sha=sha[0:7])) diff --git a/.github/template_gitref b/.github/template_gitref index 22e8d0a9..51006453 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-406-g5f397e3 +2021.08.26-417-gcec8e48 diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index c4898097..0e0a7936 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -9,7 +9,13 @@ name: "Python PR static checks" on: pull_request_target: - types: ["opened", "synchronize", "reopened"] + types: + - "opened" + - "synchronize" + - "reopened" + branches: + - "main" + - "[0-9]+.[0-9]+" # This workflow runs with elevated permissions. # Do not even think about running a single bit of code from the PR. diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index e6b6d5e3..400e0542 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -15,8 +15,4 @@ set -euv for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') do python3 .ci/scripts/validate_commit_message.py "$SHA" - VALUE=$? - if [ "$VALUE" -gt 0 ]; then - exit $VALUE - fi done diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh index bd367010..e771b5e7 100755 --- a/.github/workflows/scripts/publish_client_gem.sh +++ b/.github/workflows/scripts/publish_client_gem.sh @@ -14,22 +14,15 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. VERSION="$1" -if [[ -z "$VERSION" ]]; then +if [[ -z "${VERSION}" ]] +then echo "No version specified." exit 1 fi -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://rubygems.org/gems/pulp_python_client/versions/$VERSION")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Skipping." - exit -fi - mkdir -p ~/.gem touch ~/.gem/credentials echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials +:rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials sudo chmod 600 ~/.gem/credentials gem push "pulp_python_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index 43a31a35..144f2597 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -14,18 +14,13 @@ cd "$(dirname "$(realpath -e "$0")")/../../.." VERSION="$1" -if [[ -z "$VERSION" ]]; then +if [[ -z "${VERSION}" ]] +then echo "No version specified." exit 1 fi -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-python-client/$VERSION/")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Skipping." -else - twine upload -u __token__ -p "$PYPI_API_TOKEN" \ - "dist/pulp_python_client-$VERSION-py3-none-any.whl" \ - "dist/pulp_python-client-$VERSION.tar.gz" -fi +twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ +"dist/pulp_python_client-${VERSION}-py3-none-any.whl" \ +"dist/pulp_python-client-${VERSION}.tar.gz" \ +; diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index 384cecb2..16b0ce05 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -14,20 +14,13 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. VERSION="$1" -if [[ -z "$VERSION" ]]; then +if [[ -z "${VERSION}" ]] +then echo "No version specified." exit 1 fi -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-python/$VERSION/")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_python $VERSION has already been released. Skipping." - exit -fi - -twine upload -u __token__ -p "$PYPI_API_TOKEN" \ -dist/pulp?python-"$VERSION"-py3-none-any.whl \ -dist/pulp?python-"$VERSION".tar.gz \ +twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ +dist/pulp?python-"${VERSION}"-py3-none-any.whl \ +dist/pulp?python-"${VERSION}".tar.gz \ ; diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 02f251fa..05b93764 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -56,6 +56,7 @@ jobs: - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" + id: "create_pr_main" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_python" @@ -65,4 +66,12 @@ jobs: branch: "update-ci/main" base: "main" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_python" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_main.outputs.pull-request-number }}" + if: "steps.create_pr_main.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true ... From 40c7af9254c267c4051e93f87420efd8e4cb9338 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 19 Jan 2025 02:45:47 +0000 Subject: [PATCH 39/84] Update CI files --- .ci/ansible/settings.py.j2 | 25 +++++++++++++++++++++++-- .github/template_gitref | 2 +- .github/workflows/scripts/install.sh | 1 + template_config.yml | 3 ++- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 4ed28f42..7a73ea2f 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -27,18 +27,39 @@ API_ROOT = {{ api_root | repr }} {% endif %} {% if s3_test | default(false) %} +MEDIA_ROOT: "" +S3_USE_SIGV4 = True +{% if test_storages_compat_layer is defined and test_storages_compat_layer %} +STORAGES = { + "default": { + "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", + "OPTIONS": { + "access_key": "{{ minio_access_key }}", + "secret_key": "{{ minio_secret_key }}", + "region_name": "eu-central-1", + "addressing_style": "path", + "signature_version": "s3v4", + "bucket_name": "pulp3", + "endpoint_url": "http://minio:9000", + "default_acl": "@none None", + }, + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, +} +{% else %} DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -MEDIA_ROOT = "" AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" AWS_S3_REGION_NAME = "eu-central-1" AWS_S3_ADDRESSING_STYLE = "path" -S3_USE_SIGV4 = True AWS_S3_SIGNATURE_VERSION = "s3v4" AWS_STORAGE_BUCKET_NAME = "pulp3" AWS_S3_ENDPOINT_URL = "http://minio:9000" AWS_DEFAULT_ACL = "@none None" {% endif %} +{% endif %} {% if azure_test | default(false) %} DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" diff --git a/.github/template_gitref b/.github/template_gitref index 51006453..bdfe03f3 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-417-gcec8e48 +2021.08.26-420-gf332a34 diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index e0bf5b20..f139ebba 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -97,6 +97,7 @@ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ pulp_scenario_settings: null\ pulp_scenario_env: {}\ +test_storages_compat_layer: false\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" fi diff --git a/template_config.yml b/template_config.yml index 69d8a175..02f72d5a 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-388-g624de1a +# generated with plugin_template@2021.08.26-420-gf332a34 api_root: /pulp/ black: false @@ -65,5 +65,6 @@ test_lowerbounds: true test_performance: false test_reroute: true test_s3: true +test_storages_compat_layer: false use_issue_template: true From 0ba7fd6d38e5a184e2fa1124fe4cd817f15cd680 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Wed, 19 Feb 2025 16:30:20 -0500 Subject: [PATCH 40/84] Raise upperbound for pkginfo requirement --- CHANGES/+preliminary-md-24-support.misc | 1 + .../functional/api/test_crud_content_unit.py | 13 ++ .../tests/functional/api/test_pypi_apis.py | 136 +++++++++++------- requirements.txt | 2 +- 4 files changed, 101 insertions(+), 51 deletions(-) create mode 100644 CHANGES/+preliminary-md-24-support.misc diff --git a/CHANGES/+preliminary-md-24-support.misc b/CHANGES/+preliminary-md-24-support.misc new file mode 100644 index 00000000..7fa38ffc --- /dev/null +++ b/CHANGES/+preliminary-md-24-support.misc @@ -0,0 +1 @@ +Ensure uploading packages with metadata spec 2.4 is supported. diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index 25834bc2..f3dbe073 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -242,3 +242,16 @@ def test_upload_metadata_23_spec(python_content_factory): content = python_content_factory(filename, url=package.url) assert content.metadata_version == "2.3" break + + +@pytest.mark.parallel +def test_upload_metadata_24_spec(python_content_factory): + """Test that packages using metadata spec 2.4 can be uploaded to pulp.""" + filename = "urllib3-2.3.0-py3-none-any.whl" + with PyPISimple() as client: + page = client.get_project_page("urllib3") + for package in page.packages: + if package.filename == filename: + content = python_content_factory(filename, url=package.url) + assert content.metadata_version == "2.4" + break diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 2cab07f2..f35bb7c5 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -3,10 +3,11 @@ import requests import subprocess import tempfile +import pytest from urllib.parse import urljoin -from pulp_smash.pulp3.bindings import monitor_task, tasks as task_api +from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import get_added_content_summary, get_content_summary from pulp_python.tests.functional.constants import ( PYTHON_CONTENT_NAME, @@ -40,6 +41,32 @@ PYPI_HOST = urljoin(HOST, PULP_PYPI_BASE_URL) +@pytest.fixture +def python_empty_repo_distro(python_repo_factory, python_distribution_factory): + """Returns an empty repo with and distribution serving it.""" + def _generate_empty_repo_distro(repo_body=None, distro_body=None): + repo_body = repo_body or {} + distro_body = distro_body or {} + repo = python_repo_factory(**repo_body) + distro = python_distribution_factory(repository=repo.pulp_href, **distro_body) + return repo, distro + + yield _generate_empty_repo_distro + + +@pytest.fixture(scope="module") +def python_package_dist_directory(tmp_path_factory, http_get): + """Creates a temp dir to hold package distros for uploading.""" + dist_dir = tmp_path_factory.mktemp("dist") + egg_file = dist_dir / PYTHON_EGG_FILENAME + wheel_file = dist_dir / PYTHON_WHEEL_FILENAME + with open(egg_file, "wb") as f: + f.write(http_get(PYTHON_EGG_URL)) + with open(wheel_file, "wb") as f: + f.write(http_get(PYTHON_WHEEL_URL)) + yield dist_dir, egg_file, wheel_file + + class PyPISummaryTestCase(TestCaseUsingBindings, TestHelpersMixin): """Tests the summary response of the base url of an index.""" @@ -162,18 +189,51 @@ def test_package_upload_simple(self): content = get_added_content_summary(repo, f"{repo.versions_href}1/") self.assertDictEqual({PYTHON_CONTENT_NAME: 1}, content) - def test_twine_upload(self): - """Tests that packages can be properly uploaded through Twine.""" - repo, distro = self._create_empty_repo_and_distribution() - url = urljoin(PYPI_HOST, distro.base_path + "/legacy/") - username, password = "admin", "password" + +@pytest.mark.parallel +def test_twine_upload( + pulpcore_bindings, + python_content_summary, + python_empty_repo_distro, + python_package_dist_directory, + monitor_task, +): + """Tests that packages can be properly uploaded through Twine.""" + repo, distro = python_empty_repo_distro() + url = urljoin(distro.base_url, "legacy/") + dist_dir, _, _ = python_package_dist_directory + username, password = "admin", "password" + subprocess.run( + ( + "twine", + "upload", + "--repository-url", + url, + dist_dir / "*", + "-u", + username, + "-p", + password, + ), + capture_output=True, + check=True, + ) + tasks = pulpcore_bindings.TasksApi.list(reserved_resources=repo.pulp_href).results + for task in reversed(tasks): + t = monitor_task(task.pulp_href) + repo_ver_href = t.created_resources[-1] + summary = python_content_summary(repository_version=repo_ver_href) + assert summary.present["python.python"]["count"] == 2 + + # Test re-uploading same packages gives error + with pytest.raises(subprocess.CalledProcessError): subprocess.run( ( "twine", "upload", "--repository-url", url, - self.dists_dir.name + "/*", + dist_dir / "*", "-u", username, "-p", @@ -182,50 +242,26 @@ def test_twine_upload(self): capture_output=True, check=True, ) - tasks = task_api.list(reserved_resources_record=[repo.pulp_href]).results - for task in reversed(tasks): - t = monitor_task(task.pulp_href) - repo_ver_href = t.created_resources[-1] - content = get_content_summary(repo, f"{repo_ver_href}") - self.assertDictEqual({PYTHON_CONTENT_NAME: 2}, content) - # Test re-uploading same packages gives error - with self.assertRaises(subprocess.CalledProcessError): - subprocess.run( - ( - "twine", - "upload", - "--repository-url", - url, - self.dists_dir.name + "/*", - "-u", - username, - "-p", - password, - ), - capture_output=True, - check=True, - ) - - # Test re-uploading same packages with --skip-existing works - output = subprocess.run( - ( - "twine", - "upload", - "--repository-url", - url, - self.dists_dir.name + "/*", - "-u", - username, - "-p", - password, - "--skip-existing", - ), - capture_output=True, - check=True, - text=True - ) - self.assertEqual(output.stdout.count("Skipping"), 2) + # Test re-uploading same packages with --skip-existing works + output = subprocess.run( + ( + "twine", + "upload", + "--repository-url", + url, + dist_dir / "*", + "-u", + username, + "-p", + password, + "--skip-existing", + ), + capture_output=True, + check=True, + text=True + ) + assert output.stdout.count("Skipping") == 2 class PyPISimpleApi(TestCaseUsingBindings, TestHelpersMixin): diff --git a/requirements.txt b/requirements.txt index 35d01f9c..072eff12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ pulpcore>=3.28,<3.55 -pkginfo>=1.10.0,<1.12.0 # Twine has <1.11 in their requirements +pkginfo>=1.12.0,<1.13.0 bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 From 24f3ac11ea9c7f52008363c508307d11161e26e7 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 20 Feb 2025 05:04:49 +0000 Subject: [PATCH 41/84] Update CI files --- .ci/scripts/validate_commit_message.py | 9 +++++++-- .github/template_gitref | 2 +- .github/workflows/docs.yml | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index 04e3fd30..bb9a7d8b 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -24,7 +24,8 @@ r"^DO\s*NOT\s*MERGE", r"^EXPERIMENT", r"^FIXUP", - r"Apply suggestions from code review", + r"^fixup!", # This is created by 'git commit --fixup' + r"Apply suggestions from code review", # This usually comes from GitHub ] try: CHANGELOG_EXTS = [ @@ -40,7 +41,11 @@ if NOISSUE_MARKER in message: sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") -if any((re.match(pattern, message, re.IGNORECASE) for pattern in BLOCKING_REGEX)): +blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] +if blocking_matches: + print("Found these phrases in the commit message:") + for m in blocking_matches: + print(" - " + m.group(0)) sys.exit("This PR is not ready for consumption.") g = Github(os.environ.get("GITHUB_TOKEN")) diff --git a/.github/template_gitref b/.github/template_gitref index bdfe03f3..201b085a 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-420-gf332a34 +2021.08.26-426-g3a3f8a1 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 692ba626..d5a55a50 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,7 +13,7 @@ on: jobs: test: if: "endsWith(github.base_ref, 'main')" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-latest" defaults: run: working-directory: "pulp_python" @@ -48,7 +48,7 @@ jobs: no-test: if: "!endsWith(github.base_ref, 'main')" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-latest" steps: - run: | echo "Skip docs testing on non-main branches." From 18a0836a215940c1714ace71ac18d1cb9b014910 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 20 Feb 2025 16:49:51 +0000 Subject: [PATCH 42/84] Release 3.11.4 --- CHANGES.md | 13 +++++++++++++ CHANGES/+fix-any-type.bugfix | 2 -- CHANGES/+preliminary-md-24-support.misc | 1 - pulp_python/app/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 6 files changed, 16 insertions(+), 6 deletions(-) delete mode 100644 CHANGES/+fix-any-type.bugfix delete mode 100644 CHANGES/+preliminary-md-24-support.misc diff --git a/CHANGES.md b/CHANGES.md index 05c0de92..b016dda8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,19 @@ [//]: # (towncrier release notes start) +## 3.11.4 (2025-02-20) {: #3.11.4 } + +#### Bugfixes {: #3.11.4-bugfix } + +- Fixed the JSONField specification so it doesn't break ruby bindings. + See context [here](https://github.com/pulp/pulp_rpm/issues/3639). + +#### Misc {: #3.11.4-misc } + +- + +--- + # ## 3.11.3 (2024-08-21) {: #3.11.3 } #### Bugfixes {: #3.11.3-bugfix } diff --git a/CHANGES/+fix-any-type.bugfix b/CHANGES/+fix-any-type.bugfix deleted file mode 100644 index 89ea25f5..00000000 --- a/CHANGES/+fix-any-type.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed the JSONField specification so it doesn't break ruby bindings. -See context [here](https://github.com/pulp/pulp_rpm/issues/3639). diff --git a/CHANGES/+preliminary-md-24-support.misc b/CHANGES/+preliminary-md-24-support.misc deleted file mode 100644 index 7fa38ffc..00000000 --- a/CHANGES/+preliminary-md-24-support.misc +++ /dev/null @@ -1 +0,0 @@ -Ensure uploading packages with metadata spec 2.4 is supported. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 6ec32481..c8c0f6f6 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.4.dev" + version = "3.11.4" python_package_name = "pulp-python" diff --git a/pyproject.toml b/pyproject.toml index c6174984..13555e28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ ignore = [ [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.11.4.dev" +current_version = "3.11.4" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" diff --git a/setup.py b/setup.py index 47903f0f..a75d2591 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.4.dev", + version="3.11.4", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From 83387b5fdc3ee346fae70bc457993f33cb748ed9 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 20 Feb 2025 16:49:51 +0000 Subject: [PATCH 43/84] =?UTF-8?q?Bump=20version:=203.11.4=20=E2=86=92=203.?= =?UTF-8?q?11.5.dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pulp_python/app/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index c8c0f6f6..c2122214 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.4" + version = "3.11.5.dev" python_package_name = "pulp-python" diff --git a/pyproject.toml b/pyproject.toml index 13555e28..e2bd4283 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ ignore = [ [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.11.4" +current_version = "3.11.5.dev" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" diff --git a/setup.py b/setup.py index a75d2591..bfb4631f 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.4", + version="3.11.5.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From f839954becefdde40169d487f03d2806d2c6977f Mon Sep 17 00:00:00 2001 From: pulpbot Date: Mon, 31 Mar 2025 12:56:53 +0000 Subject: [PATCH 44/84] Update CI files --- .ci/scripts/calc_constraints.py | 4 +++- .flake8 | 3 ++- .github/template_gitref | 1 - .github/workflows/scripts/before_script.sh | 4 ++++ .github/workflows/scripts/script.sh | 2 ++ pyproject.toml | 2 +- requirements.txt | 2 +- 7 files changed, 13 insertions(+), 5 deletions(-) delete mode 100644 .github/template_gitref diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 35315153..93ee2cc7 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -83,13 +83,15 @@ def to_lower_bound(req): else: for spec in requirement.specifier: if spec.operator == ">=": + min_version = spec.version if requirement.name == "pulpcore": # Currently an exception to allow for pulpcore bugfix releases. # TODO Semver libraries should be allowed too. operator = "~=" + if len(Version(min_version).release) != 3: + raise RuntimeError("Pulpcore lower bound must be in the form '>=x.y.z'.") else: operator = "==" - min_version = spec.version return f"{requirement.name}{operator}{min_version}" return f"# NO LOWER BOUND: {req}" diff --git a/.flake8 b/.flake8 index 64403dce..9e00b3fa 100644 --- a/.flake8 +++ b/.flake8 @@ -8,13 +8,14 @@ exclude = ./docs/*,*/migrations/* per-file-ignores = */__init__.py: F401 -ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402 +ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 max-line-length = 100 # Flake8 builtin codes # -------------------- # E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed # W503: This enforces operators before line breaks which is not pep8 or black compatible. +# F824: 'nonlocal' is unused: name is never assigned in scope # Flake8-quotes extension codes # ----------------------------- diff --git a/.github/template_gitref b/.github/template_gitref deleted file mode 100644 index 201b085a..00000000 --- a/.github/template_gitref +++ /dev/null @@ -1 +0,0 @@ -2021.08.26-426-g3a3f8a1 diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 5c0072c4..11404f45 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -32,6 +32,10 @@ tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json echo "Containerfile:" tail -v -n +1 .ci/ansible/Containerfile +echo "Constraints Files:" +# The need not even exist. +tail -v -n +1 ../*/*constraints.txt || true + # Needed for some functional tests cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" cmd_prefix bash -c "usermod -a -G wheel pulp" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 8e60f87d..414d98f9 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -28,6 +28,8 @@ export PULP_URL="iframe.php?url=https%3A%2F%2Fpulp" REPORTED_STATUS="$(pulp status)" +echo "${REPORTED_STATUS}" + echo "machine pulp login admin password password diff --git a/pyproject.toml b/pyproject.toml index e2bd4283..299e1fff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,4 +71,4 @@ search = "version = \"{current_version}\"" replace = "version = \"{new_version}\"" [[tool.bumpversion.files]] -filename = "./setup.py" \ No newline at end of file +filename = "./setup.py" diff --git a/requirements.txt b/requirements.txt index 072eff12..6de2c914 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pulpcore>=3.28,<3.55 +pulpcore>=3.28.0,<3.55 pkginfo>=1.12.0,<1.13.0 bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 From 4d4a914ccda970b9d3a585b8f111283bc7c61d65 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Mon, 31 Mar 2025 17:47:50 +0200 Subject: [PATCH 45/84] Fix twine test on lower bound pulpcore version --- pulp_python/tests/functional/api/test_pypi_apis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index f35bb7c5..0bff661c 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -192,7 +192,7 @@ def test_package_upload_simple(self): @pytest.mark.parallel def test_twine_upload( - pulpcore_bindings, + tasks_api_client, python_content_summary, python_empty_repo_distro, python_package_dist_directory, @@ -218,7 +218,7 @@ def test_twine_upload( capture_output=True, check=True, ) - tasks = pulpcore_bindings.TasksApi.list(reserved_resources=repo.pulp_href).results + tasks = tasks_api_client.list(reserved_resources=repo.pulp_href).results for task in reversed(tasks): t = monitor_task(task.pulp_href) repo_ver_href = t.created_resources[-1] From cd51a952dbd8e1e558465f60d195b75c907f48cd Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 6 Apr 2025 02:54:43 +0000 Subject: [PATCH 46/84] Update CI files --- .ci/assets/ci_constraints.txt | 5 ++++- .github/workflows/scripts/script.sh | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index 2617a408..8f41e3f6 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -2,6 +2,9 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* - tablib!=3.6.0 # 3.6.0: This release introduced a regression removing the "html" optional dependency. + + +multidict!=6.3.0 +# This release failed the lower bounds test for some case sensitivity in CIMultiDict. diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 414d98f9..0faec043 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -100,8 +100,11 @@ echo "::endgroup::" # Install test requirements ########################### +# Carry on previous constraints (there might be no such file). +cat *_constraints.txt > bindings_constraints.txt || true +cat .ci/assets/ci_constraints.txt >> bindings_constraints.txt # Add a safeguard to make sure the proper versions of the clients are installed. -echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' >> bindings_constraints.txt cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt From 1d1fb02b40de2561e237667d77a2862e4f416846 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 13 Apr 2025 03:32:33 +0000 Subject: [PATCH 47/84] Update CI files --- .ci/scripts/validate_commit_message.py | 126 +++++++++++++------------ .github/workflows/ci.yml | 2 +- .github/workflows/scripts/script.sh | 10 +- 3 files changed, 74 insertions(+), 64 deletions(-) mode change 100755 => 100644 .ci/scripts/validate_commit_message.py diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py old mode 100755 new mode 100644 index bb9a7d8b..a4dc9004 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -1,66 +1,31 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template +# This file is managed by the plugin template. +# Do not edit. import os import re import subprocess import sys import tomllib +import yaml from pathlib import Path from github import Github -with open("pyproject.toml", "rb") as fp: - PYPROJECT_TOML = tomllib.load(fp) -KEYWORDS = ["fixes", "closes"] -BLOCKING_REGEX = [ - r"^DRAFT", - r"^WIP", - r"^NOMERGE", - r"^DO\s*NOT\s*MERGE", - r"^EXPERIMENT", - r"^FIXUP", - r"^fixup!", # This is created by 'git commit --fixup' - r"Apply suggestions from code review", # This usually comes from GitHub -] -try: - CHANGELOG_EXTS = [ - f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] - ] -except KeyError: - CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] -NOISSUE_MARKER = "[noissue]" - -sha = sys.argv[1] -message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - -if NOISSUE_MARKER in message: - sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") - -blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] -if blocking_matches: - print("Found these phrases in the commit message:") - for m in blocking_matches: - print(" - " + m.group(0)) - sys.exit("This PR is not ready for consumption.") -g = Github(os.environ.get("GITHUB_TOKEN")) -repo = g.get_repo("pulp/pulp_python") - - -def check_status(issue): +def check_status(issue, repo, cherry_pick): gi = repo.get_issue(int(issue)) if gi.pull_request: sys.exit(f"Error: issue #{issue} is a pull request.") - if gi.closed_at: + if gi.closed_at and not cherry_pick: + print("Make sure to use 'git cherry-pick -x' when backporting a change.") + print( + "If a backport of a change requires a significant amount of rewriting, " + "consider creating a new issue." + ) sys.exit(f"Error: issue #{issue} is closed.") -def check_changelog(issue): +def check_changelog(issue, CHANGELOG_EXTS): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -70,18 +35,63 @@ def check_changelog(issue): sys.exit(f"Invalid extension for changelog entry '{match}'.") -print("Checking commit message for {sha}.".format(sha=sha[0:7])) +def main() -> None: + TEMPLATE_CONFIG = yaml.safe_load(Path("template_config.yml").read_text()) + GITHUB_ORG = TEMPLATE_CONFIG["github_org"] + PLUGIN_NAME = TEMPLATE_CONFIG["plugin_name"] + + with Path("pyproject.toml").open("rb") as _fp: + PYPROJECT_TOML = tomllib.load(_fp) + KEYWORDS = ["fixes", "closes"] + BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"^fixup!", # This is created by 'git commit --fixup' + r"Apply suggestions from code review", # This usually comes from GitHub + ] + try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] + except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] + NOISSUE_MARKER = "[noissue]" + + sha = sys.argv[1] + message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") + + if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + + blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] + if blocking_matches: + print("Found these phrases in the commit message:") + for m in blocking_matches: + print(" - " + m.group(0)) + sys.exit("This PR is not ready for consumption.") + + g = Github(os.environ.get("GITHUB_TOKEN")) + repo = g.get_repo(f"{GITHUB_ORG}/{PLUGIN_NAME}") + + print("Checking commit message for {sha}.".format(sha=sha[0:7])) + + # validate the issue attached to the commit + issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords="|".join(KEYWORDS)) + issues = re.findall(issue_regex, message, re.IGNORECASE) + cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" + cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) + + if issues: + for issue in issues: + check_status(issue, repo, cherry_pick) + check_changelog(issue, CHANGELOG_EXTS) -# validate the issue attached to the commit -issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -issues = re.findall(issue_regex, message, re.IGNORECASE) -cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" -cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) + print("Commit message for {sha} passed.".format(sha=sha[0:7])) -if issues: - for issue in issues: - if not cherry_pick: - check_status(issue) - check_changelog(issue) -print("Commit message for {sha} passed.".format(sha=sha[0:7])) +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d82f9fee..0d16d126 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install requests pygithub + pip install requests pygithub pyyaml echo ::endgroup:: - name: "Check commit message" if: github.event_name == 'pull_request' diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 0faec043..00932968 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -119,7 +119,7 @@ echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations python --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --pyargs pulp_python.tests.unit" +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulp_python.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then @@ -135,11 +135,11 @@ if [ -f "$FUNC_TEST_SCRIPT" ]; then else if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] then - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" else - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel'" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel'" fi fi pushd ../pulp-cli From b85a3fbcb93c81a4865d1620ad57f519ed5be79a Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 15 Apr 2025 14:59:16 +0000 Subject: [PATCH 48/84] Release 3.11.5 --- CHANGES.md | 6 ++++++ pulp_python/app/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b016dda8..d73a7319 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,12 @@ [//]: # (towncrier release notes start) +## 3.11.5 (2025-04-15) {: #3.11.5 } + +No significant changes. + +--- + ## 3.11.4 (2025-02-20) {: #3.11.4 } #### Bugfixes {: #3.11.4-bugfix } diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index c2122214..67b4856e 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.5.dev" + version = "3.11.5" python_package_name = "pulp-python" diff --git a/pyproject.toml b/pyproject.toml index 299e1fff..a01ec2f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ ignore = [ [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.11.5.dev" +current_version = "3.11.5" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" diff --git a/setup.py b/setup.py index bfb4631f..265b003c 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.5.dev", + version="3.11.5", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From 265096f184cc0afb2e46317e80acb6bc50e6c9cf Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 15 Apr 2025 14:59:17 +0000 Subject: [PATCH 49/84] =?UTF-8?q?Bump=20version:=203.11.5=20=E2=86=92=203.?= =?UTF-8?q?11.6.dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pulp_python/app/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 67b4856e..90bcf070 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.5" + version = "3.11.6.dev" python_package_name = "pulp-python" diff --git a/pyproject.toml b/pyproject.toml index a01ec2f6..58d07bad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ ignore = [ [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.11.5" +current_version = "3.11.6.dev" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" diff --git a/setup.py b/setup.py index 265b003c..13ed5189 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.5", + version="3.11.6.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From 752894a271d456df7f4a81636997437b9ff35fa1 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 11 May 2025 03:00:20 +0000 Subject: [PATCH 50/84] Update CI files --- .github/workflows/docs.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d5a55a50..6144149d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,28 +22,27 @@ jobs: with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/checkout@v4" with: - python-version: "3.11" - - name: "Setup cache key" - run: | - git ls-remote https://github.com/pulp/pulp-docs main | tee pulp-docs-main-sha - - uses: "actions/cache@v4" + fetch-depth: 0 + repository: "pulp/pulp-docs" + path: "pulp-docs" + ref: "rewrite-as-mkdocs-plugin" + - uses: "actions/setup-python@v5" with: - path: "~/.cache/pip" - key: ${{ runner.os }}-pip-${{ hashFiles('pulp-docs-main-sha') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: "3.12" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install -r doc_requirements.txt + pip install ../pulp-docs towncrier echo ::endgroup:: - name: "Build changelog" run: | towncrier build --yes --version 4.0.0.ci - name: "Build docs" + working-directory: "pulp-docs" run: | + pulp-docs fetch --dest .. pulp-docs build no-test: @@ -51,4 +50,4 @@ jobs: runs-on: "ubuntu-latest" steps: - run: | - echo "Skip docs testing on non-main branches." + echo "Skip docs testing on non-default branches." From 6ee4c5c9ee6963b783e7882354f06424aee49d3f Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 1 Jun 2025 03:10:04 +0000 Subject: [PATCH 51/84] Update CI files --- .ci/scripts/check_release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index dfc4a41e..81bae1fd 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -10,8 +10,8 @@ from git import Repo RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" -Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"] -Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"] +Y_CHANGELOG_EXTS = [".feature"] +Z_CHANGELOG_EXTS = [".bugfix", ".misc"] def options(): From a12b2cf150ae56d510bd6544b3ef13b577c32852 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Mon, 16 Jun 2025 13:48:24 +0000 Subject: [PATCH 52/84] Update CI files --- .github/workflows/docs.yml | 22 ++++++++-------------- doc_requirements.txt | 2 +- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6144149d..44f17e93 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,12 +6,12 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: "Docs" +name: "Docs CI" on: workflow_call: jobs: - test: + changelog: if: "endsWith(github.base_ref, 'main')" runs-on: "ubuntu-latest" defaults: @@ -22,28 +22,22 @@ jobs: with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/checkout@v4" - with: - fetch-depth: 0 - repository: "pulp/pulp-docs" - path: "pulp-docs" - ref: "rewrite-as-mkdocs-plugin" - uses: "actions/setup-python@v5" with: python-version: "3.12" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install ../pulp-docs towncrier + pip install towncrier echo ::endgroup:: - name: "Build changelog" run: | towncrier build --yes --version 4.0.0.ci - - name: "Build docs" - working-directory: "pulp-docs" - run: | - pulp-docs fetch --dest .. - pulp-docs build + docs: + if: "endsWith(github.base_ref, 'main')" + uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@rewrite-as-mkdocs-plugin' + with: + pulpdocs_ref: 'rewrite-as-mkdocs-plugin' no-test: if: "!endsWith(github.base_ref, 'main')" diff --git a/doc_requirements.txt b/doc_requirements.txt index 6456b7d8..6daacdbe 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -5,4 +5,4 @@ # # For more info visit https://github.com/pulp/plugin_template towncrier -pulp-docs @ git+https://github.com/pulp/pulp-docs@main +pulp-docs @ git+https://github.com/pulp/pulp-docs@rewrite-as-mkdocs-plugin From fb2beccce6e2d131ef67f16b54213de282727afa Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 22 Jun 2025 03:05:08 +0000 Subject: [PATCH 53/84] Update CI files --- .github/workflows/create-branch.yml | 3 +++ .github/workflows/publish.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 9ea30712..75c3224d 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -20,6 +20,9 @@ jobs: strategy: fail-fast: false + permissions: + contents: write + steps: - uses: "actions/checkout@v4" with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 274490d9..c9505f82 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -146,6 +146,9 @@ jobs: - "publish-python-bindings" - "publish-ruby-bindings" + permissions: + contents: write + steps: - name: "Create release on GitHub" uses: "actions/github-script@v7" From 7433f61c8fb0f38c3bec3ce2894cacbddda4f00e Mon Sep 17 00:00:00 2001 From: Jitka Obselkova <41325380+jobselko@users.noreply.github.com> Date: Tue, 1 Jul 2025 17:09:38 +0200 Subject: [PATCH 54/84] Merge pull request #877 from jobselko/fix_nightly Fix tests to account for new fixtures (cherry picked from commit ee1aed2b7412b58a8803fe54a1e241ae3f6471af) --- CHANGES/876.bugfix | 1 + pulp_python/tests/functional/constants.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 CHANGES/876.bugfix diff --git a/CHANGES/876.bugfix b/CHANGES/876.bugfix new file mode 100644 index 00000000..621a1a84 --- /dev/null +++ b/CHANGES/876.bugfix @@ -0,0 +1 @@ +Fixed tests to account for new fixtures. diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index 5592dc87..d420e1ba 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -147,15 +147,16 @@ "Django", # matches 31 "pytz", # matches 6 "scipy", # matches 23 + "setuptools", # matches 2 "shelf-reader", # matches 2 ] -PYTHON_LG_PACKAGE_COUNT = 90 +PYTHON_LG_PACKAGE_COUNT = 92 PYTHON_LG_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_LG_PACKAGE_COUNT} PYTHON_LG_FIXTURE_COUNTS = { - "latest_3": 49, - "sdist": 27, - "bdist_wheel": 63, - "multi": 33, # keep_latest=1, package_types="bdist_wheel", prereleases=False + "latest_3": 51, + "sdist": 28, + "bdist_wheel": 64, + "multi": 34, # keep_latest=1, package_types="bdist_wheel", prereleases=False } DJANGO_LATEST_3 = 4 # latest version has 2 dists, each other has 1 From 9f08da7a84ca264696353770fdaa79c4cdd38ee5 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 1 Jul 2025 15:38:45 +0000 Subject: [PATCH 55/84] Update CI files --- .ci/scripts/collect_changes.py | 39 +++++++++++++++++++++++++++++----- .github/workflows/nightly.yml | 2 +- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 8cae45a9..877ebc8c 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -7,16 +7,21 @@ # For more info visit https://github.com/pulp/plugin_template import itertools +import json import os import re import tomllib +import urllib.request +from pathlib import Path from git import GitCommandError, Repo from packaging.version import parse as parse_version + +PYPI_PROJECT = "pulp_python" + # Read Towncrier settings -with open("pyproject.toml", "rb") as fp: - tc_settings = tomllib.load(fp)["tool"]["towncrier"] +tc_settings = tomllib.loads(Path("pyproject.toml").read_text())["tool"]["towncrier"] CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( @@ -35,7 +40,7 @@ # see help(re.split) for more info. NAME_REGEX = r".*" VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*" -VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})" +VERSION_CAPTURE_REGEX = rf"(?:YANKED )?({VERSION_REGEX})" DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" TITLE_REGEX = ( "(" @@ -75,6 +80,20 @@ def main(): branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True) branches = [ref.name for ref in branches] + changed = False + + try: + response = urllib.request.urlopen(f"https://pypi.org/pypi/{PYPI_PROJECT}/json") + pypi_record = json.loads(response.read()) + yanked_versions = { + parse_version(version): release[0]["yanked_reason"] + for version, release in pypi_record["releases"].items() + if release[0]["yanked"] is True + } + except Exception: + # If something failed, just don't mark anything as yanked. + yanked_versions = {} + with open(CHANGELOG_FILE, "r") as f: main_changelog = f.read() preamble, main_changes = split_changelog(main_changelog) @@ -95,9 +114,19 @@ def main(): if left[0] != right[0]: main_changes.append(right) + if yanked_versions: + for change in main_changes: + if change[0] in yanked_versions and "YANKED" not in change[1].split("\n")[0]: + reason = yanked_versions[change[0]] + version = str(change[0]) + change[1] = change[1].replace(version, "YANKED " + version, count=1) + if reason: + change[1] = change[1].replace("\n", f"\n\nYank reason: {reason}\n", count=1) + changed = True + new_length = len(main_changes) - if old_length < new_length: - print(f"{new_length - old_length} new versions have been added.") + if old_length < new_length or changed: + print(f"{new_length - old_length} new versions have been added (or something has changed).") with open(CHANGELOG_FILE, "w") as fp: fp.write(preamble) for change in main_changes: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f155d81c..ec3fcb11 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -43,7 +43,7 @@ jobs: - uses: "actions/setup-python@v5" with: - python-version: "3.11" + python-version: "3.13" - name: "Install python dependencies" run: | From 48771c2ce4bb95b58b050db8bb09efc9c77a6352 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 8 Jul 2025 15:22:31 +0000 Subject: [PATCH 56/84] Update CI files --- .ci/scripts/calc_constraints.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 93ee2cc7..66c494e9 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -53,6 +53,9 @@ def to_upper_bound(req): if requirement.name == "pulpcore": # An exception to allow for pulpcore deprecation policy. return fetch_pulpcore_upper_bound(requirement) + # skip requirement with environment scopes. E.g 'foo==1.0.0;python_version>=3.9' + if requirement.marker: + return f"# ENVIRONMENT IS UNTRACKABLE: {req}" for spec in requirement.specifier: if spec.operator == "~=": return f"# NO BETTER CONSTRAINT: {req}" From 9a2d554e1f1c5761b810a3e7b47329e1c207b287 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 13 Jul 2025 03:10:17 +0000 Subject: [PATCH 57/84] Update CI files --- .github/workflows/publish.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c9505f82..dbcd07a8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -149,11 +149,33 @@ jobs: permissions: contents: write + env: + TAG_NAME: "${{ github.ref_name }}" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + path: "pulp_python" + + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + + - name: "Install towncrier" + run: | + pip install towncrier + + - name: "Get release notes" + id: get_release_notes + run: | + NOTES=$(towncrier build --draft --version $TAG_NAME) + echo "body<> $GITHUB_OUTPUT + echo "$NOTES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: "Create release on GitHub" uses: "actions/github-script@v7" - env: - TAG_NAME: "${{ github.ref_name }}" with: script: | const { TAG_NAME } = process.env; @@ -162,5 +184,6 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, tag_name: TAG_NAME, + body: `${{ steps.get_release_notes.outputs.body }}`, make_latest: "legacy", }); From 1487d0e6144b2dd8a9f70e3933ae3178c48a12fa Mon Sep 17 00:00:00 2001 From: Jitka Obselkova <41325380+jobselko@users.noreply.github.com> Date: Tue, 22 Jul 2025 16:58:23 +0200 Subject: [PATCH 58/84] Merge pull request #908 from jobselko/fix_changelog_entry Remove changelog entry that does not impact users (cherry picked from commit 7151f91bd67852910f5a1117952ac629cc463270) --- CHANGES/876.bugfix | 1 - 1 file changed, 1 deletion(-) delete mode 100644 CHANGES/876.bugfix diff --git a/CHANGES/876.bugfix b/CHANGES/876.bugfix deleted file mode 100644 index 621a1a84..00000000 --- a/CHANGES/876.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed tests to account for new fixtures. From c9d363dc1f2ffe4f69c87517281237d9c147e441 Mon Sep 17 00:00:00 2001 From: Jitka Obselkova Date: Tue, 22 Jul 2025 17:14:18 +0200 Subject: [PATCH 59/84] Fix azure-storage-blob --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 6de2c914..b773fe8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ pulpcore>=3.28.0,<3.55 pkginfo>=1.12.0,<1.13.0 bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 +# pinned until https://github.com/Azure/Azurite/issues/2562 is resolved +azure-storage-blob<12.26.0 From 2a1e98de060512dff2e0853dfa3321eb1304eb40 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 23 Jul 2025 09:35:33 +0000 Subject: [PATCH 60/84] Release 3.11.6 --- CHANGES.md | 6 ++++++ pulp_python/app/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d73a7319..b5a043bd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,12 @@ [//]: # (towncrier release notes start) +## 3.11.6 (2025-07-23) {: #3.11.6 } + +No significant changes. + +--- + ## 3.11.5 (2025-04-15) {: #3.11.5 } No significant changes. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 90bcf070..74ffbff9 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.6.dev" + version = "3.11.6" python_package_name = "pulp-python" diff --git a/pyproject.toml b/pyproject.toml index 58d07bad..ebd0261d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ ignore = [ [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.11.6.dev" +current_version = "3.11.6" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" diff --git a/setup.py b/setup.py index 13ed5189..e9967ca6 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.6.dev", + version="3.11.6", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From f4aa91edeab05b18490e1af2681641021f1587b5 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 23 Jul 2025 09:35:34 +0000 Subject: [PATCH 61/84] =?UTF-8?q?Bump=20version:=203.11.6=20=E2=86=92=203.?= =?UTF-8?q?11.7.dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pulp_python/app/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 74ffbff9..943ecbdc 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.6" + version = "3.11.7.dev" python_package_name = "pulp-python" diff --git a/pyproject.toml b/pyproject.toml index ebd0261d..fdda2933 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ ignore = [ [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.11.6" +current_version = "3.11.7.dev" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" diff --git a/setup.py b/setup.py index e9967ca6..6f79d175 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.6", + version="3.11.7.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From ab6bf1d6bf39e1ef06d49b9a4f953105ace65865 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 3 Aug 2025 03:13:57 +0000 Subject: [PATCH 62/84] Update CI files --- .ci/ansible/Containerfile.j2 | 6 +++--- template_config.yml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index d57b44ed..01469b64 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -5,9 +5,9 @@ FROM {{ ci_base | default(pulp_default_container) }} ADD ./{{ item.name }} ./{{ item.name }} {% endfor %} -# Install python packages -# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads -# Hacking botocore (https://github.com/boto/botocore/pull/1990) +{% for item in extra_files | default([]) %} +ADD ./{{ item.origin }} {{ item.destination }} +{% endfor %} # This MUST be the ONLY call to pip install in inside the container. RUN pip3 install --upgrade pip setuptools wheel && \ diff --git a/template_config.yml b/template_config.yml index 02f72d5a..b75794d1 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-420-gf332a34 +# generated with plugin_template api_root: /pulp/ black: false @@ -20,6 +20,7 @@ deploy_client_to_rubygems: true deploy_to_pypi: true disabled_redis_runners: [] docker_fixtures: false +extra_files: [] flake8: true flake8_ignore: [] github_org: pulp From 6acab1577df1253b2153437280e7781f459c7fdb Mon Sep 17 00:00:00 2001 From: Jitka Obselkova <41325380+jobselko@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:54:41 +0200 Subject: [PATCH 63/84] Merge pull request #924 from jobselko/3_10_unpin_azure_storage_blob Unpin azure-storage-blob (cherry picked from commit 92dd71cf17d843ab04a9c88c919f9d5f43eae8ef) --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b773fe8b..6de2c914 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,3 @@ pulpcore>=3.28.0,<3.55 pkginfo>=1.12.0,<1.13.0 bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 -# pinned until https://github.com/Azure/Azurite/issues/2562 is resolved -azure-storage-blob<12.26.0 From 0b02e4ae2a1ac8d896abd2bc81796acf843b3e5a Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 21 Aug 2025 16:17:02 -0400 Subject: [PATCH 64/84] Update Azure and S3 storage settings for 3.11 Set legacy storage settings in template_config.yml based on pulpcore requirement (>=3.28.0,<3.55). This addresses the recent change where plugin-template stopped automatically setting test scenario storage settings and now requires plugins to define them in their template_config.yml. --- .ci/ansible/settings.py.j2 | 53 ---------------------------- .github/workflows/scripts/install.sh | 5 ++- template_config.yml | 24 +++++++++++-- 3 files changed, 23 insertions(+), 59 deletions(-) diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 7a73ea2f..dfe2851d 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -26,56 +26,3 @@ API_ROOT = {{ api_root | repr }} {% endfor %} {% endif %} -{% if s3_test | default(false) %} -MEDIA_ROOT: "" -S3_USE_SIGV4 = True -{% if test_storages_compat_layer is defined and test_storages_compat_layer %} -STORAGES = { - "default": { - "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", - "OPTIONS": { - "access_key": "{{ minio_access_key }}", - "secret_key": "{{ minio_secret_key }}", - "region_name": "eu-central-1", - "addressing_style": "path", - "signature_version": "s3v4", - "bucket_name": "pulp3", - "endpoint_url": "http://minio:9000", - "default_acl": "@none None", - }, - }, - "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", - }, -} -{% else %} -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" -AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" -AWS_S3_REGION_NAME = "eu-central-1" -AWS_S3_ADDRESSING_STYLE = "path" -AWS_S3_SIGNATURE_VERSION = "s3v4" -AWS_STORAGE_BUCKET_NAME = "pulp3" -AWS_S3_ENDPOINT_URL = "http://minio:9000" -AWS_DEFAULT_ACL = "@none None" -{% endif %} -{% endif %} - -{% if azure_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" -MEDIA_ROOT = "" -AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -AZURE_ACCOUNT_NAME = "devstoreaccount1" -AZURE_CONTAINER = "pulp-test" -AZURE_LOCATION = "pulp3" -AZURE_OVERWRITE_FILES = True -AZURE_URL_EXPIRATION_SECS = 120 -AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' -{% endif %} - -{% if gcp_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" -MEDIA_ROOT = "" -GS_BUCKET_NAME = "gcppulp" -GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443" -{% endif %} diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index f139ebba..0103c089 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -95,9 +95,8 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": ""}\ pulp_scenario_env: {}\ -test_storages_compat_layer: false\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" fi @@ -110,7 +109,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": ""}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/template_config.yml b/template_config.yml index b75794d1..33c3d396 100644 --- a/template_config.yml +++ b/template_config.yml @@ -46,9 +46,28 @@ pulp_settings: allowed_import_paths: /tmp orphan_protection_time: 0 pypi_api_hostname: https://pulp:443 -pulp_settings_azure: null +pulp_settings_azure: + AZURE_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + AZURE_ACCOUNT_NAME: devstoreaccount1 + AZURE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; + AZURE_CONTAINER: pulp-test + AZURE_LOCATION: pulp3 + AZURE_OVERWRITE_FILES: true + AZURE_URL_EXPIRATION_SECS: 120 + DEFAULT_FILE_STORAGE: storages.backends.azure_storage.AzureStorage + MEDIA_ROOT: '' pulp_settings_gcp: null -pulp_settings_s3: null +pulp_settings_s3: + AWS_ACCESS_KEY_ID: AKIAIT2Z5TDYPX3ARJBA + AWS_DEFAULT_ACL: '@none None' + AWS_S3_ADDRESSING_STYLE: path + AWS_S3_ENDPOINT_URL: http://minio:9000 + AWS_S3_REGION_NAME: eu-central-1 + AWS_S3_SIGNATURE_VERSION: s3v4 + AWS_SECRET_ACCESS_KEY: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS + AWS_STORAGE_BUCKET_NAME: pulp3 + DEFAULT_FILE_STORAGE: storages.backends.s3boto3.S3Boto3Storage + MEDIA_ROOT: '' pydocstyle: true release_email: pulp-infra@redhat.com release_user: pulpbot @@ -66,6 +85,5 @@ test_lowerbounds: true test_performance: false test_reroute: true test_s3: true -test_storages_compat_layer: false use_issue_template: true From 1988607ef2ba2c8e28ac5164d78dad7af66d6337 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 31 Aug 2025 02:55:58 +0000 Subject: [PATCH 65/84] Update CI files --- .github/workflows/docs.yml | 4 ++-- doc_requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 44f17e93..1f9fe5f0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,9 +35,9 @@ jobs: towncrier build --yes --version 4.0.0.ci docs: if: "endsWith(github.base_ref, 'main')" - uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@rewrite-as-mkdocs-plugin' + uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@main' with: - pulpdocs_ref: 'rewrite-as-mkdocs-plugin' + pulpdocs_ref: 'main' no-test: if: "!endsWith(github.base_ref, 'main')" diff --git a/doc_requirements.txt b/doc_requirements.txt index 6daacdbe..6456b7d8 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -5,4 +5,4 @@ # # For more info visit https://github.com/pulp/plugin_template towncrier -pulp-docs @ git+https://github.com/pulp/pulp-docs@rewrite-as-mkdocs-plugin +pulp-docs @ git+https://github.com/pulp/pulp-docs@main From b6062655d4164d17c3592e948b54c023c0781599 Mon Sep 17 00:00:00 2001 From: Grant Gainey Date: Mon, 8 Sep 2025 16:36:34 -0400 Subject: [PATCH 66/84] Merge pull request #957 from gerrod3/skip-exist-remove Remove testing of twine's --skip-existing due to twine feature removal (cherry picked from commit 2b34643299835462a4add6b3b76d2d8309bf91d9) --- .../tests/functional/api/test_pypi_apis.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 0bff661c..7e6fa626 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -243,26 +243,6 @@ def test_twine_upload( check=True, ) - # Test re-uploading same packages with --skip-existing works - output = subprocess.run( - ( - "twine", - "upload", - "--repository-url", - url, - dist_dir / "*", - "-u", - username, - "-p", - password, - "--skip-existing", - ), - capture_output=True, - check=True, - text=True - ) - assert output.stdout.count("Skipping") == 2 - class PyPISimpleApi(TestCaseUsingBindings, TestHelpersMixin): """Tests that the simple api is correct.""" From 470cef857f92d8e8e1fc285f3bbf14f741146b3e Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 14 Sep 2025 02:52:12 +0000 Subject: [PATCH 67/84] Update CI files --- .ci/scripts/skip_tests.py | 113 ++++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 51 ++++++++++++++++- 2 files changed, 162 insertions(+), 2 deletions(-) create mode 100755 .ci/scripts/skip_tests.py diff --git a/.ci/scripts/skip_tests.py b/.ci/scripts/skip_tests.py new file mode 100755 index 00000000..2a21a723 --- /dev/null +++ b/.ci/scripts/skip_tests.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +""" +skip_tests.py - Check if only documentation files were changed in a git branch + +Usage: + ./skip_tests.py + +Arguments: + git_root: The root directory of the git project + reference_branch: The branch to compare against + +Returns: + 0: Skip + 1: NoSkip + *: Error +""" + +import sys +import os +import re +import git +import textwrap +import argparse + +DOC_PATTERNS = [ + r"^docs/", + r"\.md$", + r"\.txt$", + r"LICENSE.*", + r"CHANGELOG.*", + r"CHANGES.*", + r"CONTRIBUTING.*", +] + +# Exit codes +CODE_SKIP = 0 +CODE_NO_SKIP = 1 +CODE_ERROR = 2 + + +def main() -> int: + git_root, reference_branch = get_args() + changed_files = get_changed_files(git_root, reference_branch) + if not changed_files: + return CODE_SKIP + doc_files = [f for f in changed_files if is_doc_file(f)] + not_doc_files = set(changed_files) - set(doc_files) + print_changes(doc_files, not_doc_files) + if not_doc_files: + return CODE_NO_SKIP + else: + return CODE_SKIP + + +# Utils + + +def get_changed_files(git_root: str, reference_branch: str) -> list[str]: + """Get list of files changed between current branch and reference branch.""" + repo = git.Repo(git_root) + diff_index = repo.git.diff("--name-only", reference_branch).strip() + if not diff_index: + return [] + return [f.strip() for f in diff_index.split("\n") if f.strip()] + + +def is_doc_file(file_path: str) -> bool: + """Check if a file is a documentation file.""" + for pattern in DOC_PATTERNS: + if re.search(pattern, file_path): + return True + return False + + +def print_changes(doc_files: list[str], not_doc_files: list[str]) -> None: + display_doc = " \n".join(doc_files) + print(f"doc_files({len(doc_files)})") + if doc_files: + display_doc = "\n".join(doc_files) + print(textwrap.indent(display_doc, " ")) + + print(f"non_doc_files({len(not_doc_files)})") + if not_doc_files: + display_non_doc = " \n".join(not_doc_files) + print(textwrap.indent(display_non_doc, " ")) + + +def get_args() -> tuple[str, str]: + """Parse command line arguments and validate them.""" + parser = argparse.ArgumentParser(description="Check if CI can skip tests for a git branch") + parser.add_argument("git_root", help="The root directory of the git project") + parser.add_argument("reference_branch", help="The branch to compare against") + args = parser.parse_args() + git_root = os.path.abspath(args.git_root) + ref_branch = args.reference_branch + + if not os.path.exists(git_root): + raise ValueError(f"Git root directory does not exist: {git_root}") + if not os.path.isdir(git_root): + raise ValueError(f"Git root is not a directory: {git_root}") + try: + git.Repo(git_root) + except git.InvalidGitRepositoryError: + raise ValueError(f"Directory is not a git repository: {git_root}") + return git_root, ref_branch + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + print(e) + sys.exit(CODE_ERROR) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d16d126..a7817dfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,10 +43,49 @@ jobs: run: | .github/workflows/scripts/check_commit.sh + check-changes: + runs-on: ubuntu-latest + outputs: + run_tests: ${{ steps.check.outputs.run_tests }} + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + path: "pulp_python" + + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install gitpython + echo ::endgroup:: + + - name: Analyze changed files + shell: bash + id: check + run: | + set +e + BASE_REF=${{ github.event.pull_request.base.sha }} + echo "Checking against:" + git name-rev $BASE_REF + python3 .ci/scripts/skip_tests.py . $BASE_REF + exit_code=$? + if [ $exit_code -ne 0 ] && [ $exit_code -ne 1 ]; then + echo "Error: skip_tests.py returned unexpected exit code $exit_code" + exit $exit_code + fi + echo "run_tests=$exit_code" >> $GITHUB_OUTPUT + docs: uses: "./.github/workflows/docs.yml" lint: + needs: + - "check-changes" + if: needs.check-changes.outputs.run_tests == '1' uses: "./.github/workflows/lint.yml" build: @@ -84,6 +123,7 @@ jobs: # This is a dummy dependent task to have a single entry for the branch protection rules. runs-on: "ubuntu-latest" needs: + - "check-changes" - "check-commits" - "lint" - "test" @@ -93,6 +133,13 @@ jobs: - name: "Collect needed jobs results" working-directory: "." run: | - echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result' - echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0' + if [ ${{ needs.check-changes.outputs.run_tests }} == "1" ]; then + # Full test run - check all jobs + echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result' + echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0' + else + # Docs-only run - check only required jobs (exclude lint and test) + echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.key != "lint" and .key != "test")|select(.value.result!="success")|.key + ": " + .value.result' + echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.key != "lint" and .key != "test"))|map(select(.value.result!="success"))|length == 0' + fi echo "CI says: Looks good!" From d2e76903186423970cc6b8550e93fb6ca8976a77 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 21 Sep 2025 02:56:00 +0000 Subject: [PATCH 68/84] Update CI files --- .github/workflows/release.yml | 4 +++- .github/workflows/test.yml | 2 +- lint_requirements.txt | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b758bae6..7291dadd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,10 +32,12 @@ jobs: with: python-version: "3.11" + # Click is pinned because of: + # https://github.com/pallets/click/issues/3065 - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump-my-version towncrier + pip install bump-my-version towncrier 'click<8.3' echo ::endgroup:: - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a18b9230..02e0c673 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,7 +71,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version + pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version 'click<8.3' echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: diff --git a/lint_requirements.txt b/lint_requirements.txt index a2936209..092ccbd0 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,6 +5,9 @@ # # For more info visit https://github.com/pulp/plugin_template +# Click is pinned because of: +# https://github.com/pallets/click/issues/3065 +click<8.3 bump-my-version check-manifest flake8 From 0a7bc7afba3db19877d1b5a142c62960f14e407a Mon Sep 17 00:00:00 2001 From: pulpbot Date: Mon, 20 Oct 2025 15:56:54 +0000 Subject: [PATCH 69/84] Update CI files --- .ci/scripts/skip_tests.py | 1 - .github/workflows/publish.yml | 5 ++++- .github/workflows/scripts/build_python_client.sh | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.ci/scripts/skip_tests.py b/.ci/scripts/skip_tests.py index 2a21a723..a68d000d 100755 --- a/.ci/scripts/skip_tests.py +++ b/.ci/scripts/skip_tests.py @@ -25,7 +25,6 @@ DOC_PATTERNS = [ r"^docs/", r"\.md$", - r"\.txt$", r"LICENSE.*", r"CHANGELOG.*", r"CHANGES.*", diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dbcd07a8..018013ad 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -167,8 +167,11 @@ jobs: pip install towncrier - name: "Get release notes" - id: get_release_notes + id: "get_release_notes" + shell: "bash" run: | + # The last commit before the release commit contains the release CHANGES fragments + git checkout "${TAG_NAME}~" NOTES=$(towncrier build --draft --version $TAG_NAME) echo "body<> $GITHUB_OUTPUT echo "$NOTES" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh index e86ffc29..53db9132 100755 --- a/.github/workflows/scripts/build_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -24,7 +24,7 @@ pushd pulp_python-client python setup.py sdist bdist_wheel --python-tag py3 twine check "dist/pulp_python_client-"*"-py3-none-any.whl" -twine check "dist/pulp_python-client-"*".tar.gz" +twine check "dist/pulp_python_client-"*".tar.gz" tar cvf "../../pulp_python/python-python-client.tar" ./dist From c10aeb277e2f24eab834c2b86b1be8b426c85fc6 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 21 Oct 2025 21:39:44 +0000 Subject: [PATCH 70/84] Update CI files --- .github/workflows/ci.yml | 44 +++++++++++++++---- .github/workflows/docs.yml | 15 +++---- .../workflows/scripts/build_python_client.sh | 2 +- .../workflows/scripts/publish_client_pypi.sh | 2 +- .github/workflows/scripts/script.sh | 2 +- .github/workflows/test.yml | 2 +- 6 files changed, 46 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7817dfa..0e9afb1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,7 @@ jobs: runs-on: ubuntu-latest outputs: run_tests: ${{ steps.check.outputs.run_tests }} + run_docs: ${{ steps.check.outputs.run_docs }} steps: - uses: "actions/checkout@v4" with: @@ -67,6 +68,13 @@ jobs: shell: bash id: check run: | + # We only test docs on the default branch (usually main) + if [[ "${{ github.base_ref }}" == *"main" ]]; then + echo "run_docs=1" >> $GITHUB_OUTPUT + else + echo "run_docs=0" >> $GITHUB_OUTPUT + fi + set +e BASE_REF=${{ github.event.pull_request.base.sha }} echo "Checking against:" @@ -80,7 +88,10 @@ jobs: echo "run_tests=$exit_code" >> $GITHUB_OUTPUT docs: + needs: "check-changes" uses: "./.github/workflows/docs.yml" + with: + run_docs: ${{ needs.check-changes.outputs.run_docs }} lint: needs: @@ -133,13 +144,30 @@ jobs: - name: "Collect needed jobs results" working-directory: "." run: | - if [ ${{ needs.check-changes.outputs.run_tests }} == "1" ]; then - # Full test run - check all jobs - echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result' - echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0' - else - # Docs-only run - check only required jobs (exclude lint and test) - echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.key != "lint" and .key != "test")|select(.value.result!="success")|.key + ": " + .value.result' - echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.key != "lint" and .key != "test"))|map(select(.value.result!="success"))|length == 0' + RUN_TESTS=${{ needs.check-changes.outputs.run_tests }} + RUN_DOCS=${{ needs.check-changes.outputs.run_docs }} + + check_jobs() { + local filter="$1" + local needs_json='${{toJson(needs)}}' + # output failed jobs after filter + echo "$needs_json" | jq -r "to_entries[]|select($filter)|select(.value.result!=\"success\")|.key + \": \" + .value.result" + # fails if not all selected jobs passed + echo "$needs_json" | jq -e "to_entries|map(select($filter))|map(select(.value.result!=\"success\"))|length == 0" + } + + if [ "$RUN_TESTS" == "1" ] && [ "$RUN_DOCS" == "1" ]; then + FILTERS="true" # check all jobs + elif [ "$RUN_TESTS" == "1" ] && [ "$RUN_DOCS" == "0" ]; then + echo "Skipping docs: running on non-default branch" + FILTERS='.key != "docs"' + elif [ "$RUN_TESTS" == "0" ] && [ "$RUN_DOCS" == "1" ]; then + echo "Skipping tests: only doc changes" + FILTERS='.key != "lint" and .key != "test"' + else # RUN_TESTS=0, RUN_DOCS=0 + echo "What is this PR doing??" + FILTERS='.key != "lint" and .key != "test" and .key != "docs"' fi + + check_jobs "$FILTERS" echo "CI says: Looks good!" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1f9fe5f0..9e16317d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,10 +9,14 @@ name: "Docs CI" on: workflow_call: + inputs: + run_docs: + description: "Whether to run docs jobs" + required: true + type: string jobs: changelog: - if: "endsWith(github.base_ref, 'main')" runs-on: "ubuntu-latest" defaults: run: @@ -34,14 +38,7 @@ jobs: run: | towncrier build --yes --version 4.0.0.ci docs: - if: "endsWith(github.base_ref, 'main')" + if: ${{ inputs.run_docs == '1' }} uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@main' with: pulpdocs_ref: 'main' - - no-test: - if: "!endsWith(github.base_ref, 'main')" - runs-on: "ubuntu-latest" - steps: - - run: | - echo "Skip docs testing on non-default branches." diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh index 53db9132..9eb0def9 100755 --- a/.github/workflows/scripts/build_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -21,7 +21,7 @@ rm -rf "pulp_python-client" ./gen-client.sh "../pulp_python/python-api.json" "python" python "pulp_python" pushd pulp_python-client -python setup.py sdist bdist_wheel --python-tag py3 +python -m build twine check "dist/pulp_python_client-"*"-py3-none-any.whl" twine check "dist/pulp_python_client-"*".tar.gz" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index 144f2597..c5a4295c 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -22,5 +22,5 @@ fi twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ "dist/pulp_python_client-${VERSION}-py3-none-any.whl" \ -"dist/pulp_python-client-${VERSION}.tar.gz" \ +"dist/pulp_python_client-${VERSION}.tar.gz" \ ; diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 00932968..0770527d 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -72,7 +72,7 @@ pushd ../pulp-openapi-generator rm -rf "./${PACKAGE}-client" ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" pushd "${PACKAGE}-client" - python setup.py sdist bdist_wheel --python-tag py3 + python -m build popd else if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 02e0c673..7a244d25 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,7 +71,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version 'click<8.3' + pip install build towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version 'click<8.3' echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: From 1470106442530dfd6e17dbcb65a834e0bf9be806 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 18 Nov 2025 15:23:47 +0000 Subject: [PATCH 71/84] Release 3.11.7 --- CHANGES.md | 6 ++++++ pulp_python/app/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b5a043bd..8190f38b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,12 @@ [//]: # (towncrier release notes start) +## 3.11.7 (2025-11-18) {: #3.11.7 } + +No significant changes. + +--- + ## 3.11.6 (2025-07-23) {: #3.11.6 } No significant changes. diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 943ecbdc..6d2d0b51 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.7.dev" + version = "3.11.7" python_package_name = "pulp-python" diff --git a/pyproject.toml b/pyproject.toml index fdda2933..ad69e3e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ ignore = [ [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.11.7.dev" +current_version = "3.11.7" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" diff --git a/setup.py b/setup.py index 6f79d175..396b132c 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.7.dev", + version="3.11.7", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From 173f541b7a5ff5afdbd6211994a5c454163a55c7 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 18 Nov 2025 15:23:47 +0000 Subject: [PATCH 72/84] =?UTF-8?q?Bump=20version:=203.11.7=20=E2=86=92=203.?= =?UTF-8?q?11.8.dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pulp_python/app/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index 6d2d0b51..d008fbc2 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): name = "pulp_python.app" label = "python" - version = "3.11.7" + version = "3.11.8.dev" python_package_name = "pulp-python" diff --git a/pyproject.toml b/pyproject.toml index ad69e3e7..ec98c6e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ ignore = [ [tool.bumpversion] # This section is managed by the plugin template. Do not edit manually. -current_version = "3.11.7" +current_version = "3.11.8.dev" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" diff --git a/setup.py b/setup.py index 396b132c..ed82a6e7 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="pulp-python", - version="3.11.7", + version="3.11.8.dev", description="pulp-python plugin for the Pulp Project", long_description=long_description, long_description_content_type="text/markdown", From 6f0978095aea1e833ff7490b7f3bdcb7e185d309 Mon Sep 17 00:00:00 2001 From: Gerrod Date: Wed, 19 Nov 2025 16:19:35 -0500 Subject: [PATCH 73/84] Merge pull request #1023 from gerrod3/fixt-pep-updates Update large fixtures count (cherry picked from commit c7cfdde34b5dfcb855530144da4b1ac3ee0ba176) --- pulp_python/tests/functional/constants.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index d420e1ba..f0e53046 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -144,19 +144,22 @@ "aiohttp", # matches 7 "bcrypt", # matches 8 "celery", # matches 13 + "crane", # matches 0 "Django", # matches 31 + "pulp-2to3-migration", # matches 2 "pytz", # matches 6 "scipy", # matches 23 "setuptools", # matches 2 "shelf-reader", # matches 2 + "twine", # matches 14 ] -PYTHON_LG_PACKAGE_COUNT = 92 +PYTHON_LG_PACKAGE_COUNT = 108 PYTHON_LG_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_LG_PACKAGE_COUNT} PYTHON_LG_FIXTURE_COUNTS = { - "latest_3": 51, - "sdist": 28, - "bdist_wheel": 64, - "multi": 34, # keep_latest=1, package_types="bdist_wheel", prereleases=False + "latest_3": 59, + "sdist": 36, + "bdist_wheel": 72, + "multi": 36, # keep_latest=1, package_types="bdist_wheel", prereleases=False } DJANGO_LATEST_3 = 4 # latest version has 2 dists, each other has 1 From bdc2c56803fb5f7b2fdfca68001d865fab1145bc Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Wed, 19 Nov 2025 16:30:16 -0500 Subject: [PATCH 74/84] Fix pip install test --- pulp_python/tests/functional/api/test_consume_content.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pulp_python/tests/functional/api/test_consume_content.py b/pulp_python/tests/functional/api/test_consume_content.py index dca40728..a46358be 100644 --- a/pulp_python/tests/functional/api/test_consume_content.py +++ b/pulp_python/tests/functional/api/test_consume_content.py @@ -122,6 +122,7 @@ def install(cli_client, package, host=PYPI_URL): "pip", "install", "--no-deps", + "--no-build-isolation", "--trusted-host", urlsplit(host).hostname, "--trusted-host", From 8369132c0ce75f7bcc2f3e15c01b31b84ca50388 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 30 Nov 2025 03:04:27 +0000 Subject: [PATCH 75/84] Update CI files --- .github/workflows/publish.yml | 88 ++++++------------- .github/workflows/release.yml | 4 +- .../workflows/scripts/publish_client_gem.sh | 28 ------ .../workflows/scripts/publish_client_pypi.sh | 26 ------ .../workflows/scripts/publish_plugin_pypi.sh | 26 ------ .github/workflows/test.yml | 2 +- lint_requirements.txt | 3 - 7 files changed, 29 insertions(+), 148 deletions(-) delete mode 100755 .github/workflows/scripts/publish_client_gem.sh delete mode 100755 .github/workflows/scripts/publish_client_pypi.sh delete mode 100755 .github/workflows/scripts/publish_plugin_pypi.sh diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 018013ad..525144fa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,47 +23,28 @@ jobs: runs-on: "ubuntu-latest" needs: - "build" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + environment: + name: "pypi" + url: "https://pypi.org/p/pulp-python" + permissions: + id-token: "write" steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_python" - - uses: "actions/download-artifact@v4" with: name: "plugin_package" - path: "pulp_python/dist/" + path: "dist/" - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Deploy plugin to pypi" - run: | - .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.ref_name }} + - name: "Publish package to PyPI" + uses: pypa/gh-action-pypi-publish@release/v1 publish-python-bindings: runs-on: "ubuntu-latest" needs: - "build" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + environment: + name: "pypi" + permissions: + id-token: "write" steps: - uses: "actions/checkout@v4" @@ -81,32 +62,18 @@ jobs: run: | tar -xvf python-python-client.tar - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: "Publish client to pypi" - run: | - bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.ref_name }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: "pulp_python/dist/" publish-ruby-bindings: runs-on: "ubuntu-latest" needs: - "build" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + environment: + name: "rubygems" + permissions: + id-token: "write" steps: - uses: "actions/checkout@v4" @@ -128,15 +95,12 @@ jobs: with: ruby-version: "2.6" - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" + - name: "Set RubyGems Credentials" + uses: "rubygems/configure-rubygems-credentials@v1.0.0" - - name: "Publish client to rubygems" + - name: "Publish client to RubyGems" run: | - bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} + gem push "pulp_python_client-${{ github.ref_name }}.gem" create-gh-release: runs-on: "ubuntu-latest" @@ -179,14 +143,16 @@ jobs: - name: "Create release on GitHub" uses: "actions/github-script@v7" + env: + RELEASE_BODY: ${{ steps.get_release_notes.outputs.body }} with: script: | - const { TAG_NAME } = process.env; + const { TAG_NAME, RELEASE_BODY } = process.env; await github.rest.repos.createRelease({ owner: context.repo.owner, repo: context.repo.repo, tag_name: TAG_NAME, - body: `${{ steps.get_release_notes.outputs.body }}`, + body: RELEASE_BODY, make_latest: "legacy", }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7291dadd..b758bae6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,12 +32,10 @@ jobs: with: python-version: "3.11" - # Click is pinned because of: - # https://github.com/pallets/click/issues/3065 - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump-my-version towncrier 'click<8.3' + pip install bump-my-version towncrier echo ::endgroup:: - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh deleted file mode 100755 index e771b5e7..00000000 --- a/.github/workflows/scripts/publish_client_gem.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -VERSION="$1" - -if [[ -z "${VERSION}" ]] -then - echo "No version specified." - exit 1 -fi - -mkdir -p ~/.gem -touch ~/.gem/credentials -echo "--- -:rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials -sudo chmod 600 ~/.gem/credentials -gem push "pulp_python_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh deleted file mode 100755 index c5a4295c..00000000 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -VERSION="$1" - -if [[ -z "${VERSION}" ]] -then - echo "No version specified." - exit 1 -fi - -twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ -"dist/pulp_python_client-${VERSION}-py3-none-any.whl" \ -"dist/pulp_python_client-${VERSION}.tar.gz" \ -; diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh deleted file mode 100755 index 16b0ce05..00000000 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -VERSION="$1" - -if [[ -z "${VERSION}" ]] -then - echo "No version specified." - exit 1 -fi - -twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ -dist/pulp?python-"${VERSION}"-py3-none-any.whl \ -dist/pulp?python-"${VERSION}".tar.gz \ -; diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7a244d25..88cfd8f3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,7 +71,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install build towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version 'click<8.3' + pip install build towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: diff --git a/lint_requirements.txt b/lint_requirements.txt index 092ccbd0..a2936209 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,9 +5,6 @@ # # For more info visit https://github.com/pulp/plugin_template -# Click is pinned because of: -# https://github.com/pallets/click/issues/3065 -click<8.3 bump-my-version check-manifest flake8 From 47d7b45f1096dbe423e6ce2743febb709c102229 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 8 Jan 2026 15:38:28 +0000 Subject: [PATCH 76/84] Update CI files --- .ci/assets/ci_constraints.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index 8f41e3f6..9e941e96 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -8,3 +8,7 @@ tablib!=3.6.0 multidict!=6.3.0 # This release failed the lower bounds test for some case sensitivity in CIMultiDict. + + +azure-storage-blob!=12.28.* +# Apperently does not work with current azurite. From bf85729d2108b2bfd955ebd0e80a0b2a9be4a1c1 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Mon, 12 Jan 2026 15:37:48 +0000 Subject: [PATCH 77/84] Update CI files --- .ci/assets/ci_constraints.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index 9e941e96..c8070aff 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -11,4 +11,8 @@ multidict!=6.3.0 azure-storage-blob!=12.28.* -# Apperently does not work with current azurite. +# Apparently does not work with current azurite. + + +pycares<5 +# older aiodns versions don't pin pycares UB, and are broken by pycares>=5 From 53c5d307f30b46a41668679c2e82dbcaaff57987 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 25 Jan 2026 03:14:07 +0000 Subject: [PATCH 78/84] Update CI files --- .github/workflows/scripts/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 0103c089..87a53dae 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -107,7 +107,7 @@ if [ "$TEST" = "azure" ]; then image: mcr.microsoft.com/azure-storage/azurite\ volumes:\ - ./azurite:/etc/pulp\ - command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml + command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": ""}\ pulp_scenario_env: {}\ From 8e03064fe9631b89b55641a26a07157f45253320 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 8 Feb 2026 03:36:24 +0000 Subject: [PATCH 79/84] Update CI files --- .github/pull_request_template.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..4bee60fa --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,15 @@ + + +### 📜 Checklist + +- [ ] Commits are cleanly separated with meaningful messages (simple features and bug fixes should be [squashed](https://pulpproject.org/pulpcore/docs/dev/guides/git/#rebasing-and-squashing) to one commit) +- [ ] A [changelog entry](https://pulpproject.org/pulpcore/docs/dev/guides/git/#changelog-update) or entries has been added for any significant changes +- [ ] Follows the [Pulp policy on AI Usage](https://pulpproject.org/help/more/governance/ai_policy/) +- [ ] (For new features) - User documentation and test coverage has been added + +See: [Pull Request Walkthrough](https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/) From 5013d16cfb06e2954aaa54eb5171d057cc1bcc35 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Sun, 22 Feb 2026 20:00:26 -0500 Subject: [PATCH 80/84] Migrate to Django 5 storages --- .ci/assets/ci_constraints.txt | 3 +- .ci/scripts/check_release.py | 11 +++- .github/workflows/ci.yml | 8 +-- .github/workflows/scripts/install.sh | 4 +- pulp_python/app/models.py | 4 +- pulp_python/app/utils.py | 5 +- template_config.yml | 88 ++++++++++++++++------------ 7 files changed, 72 insertions(+), 51 deletions(-) diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index c8070aff..c9198f19 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -1,5 +1,6 @@ # Pulpcore versions without the openapi command do no longer work in the CI -pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* +# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70 +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.* tablib!=3.6.0 diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 81bae1fd..86e250e4 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -1,11 +1,20 @@ #!/usr/bin/env python +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# "pyyaml>=6.0.3,<6.1.0", +# ] +# /// import argparse import re import os import tomllib -import yaml from pathlib import Path + +import yaml from packaging.version import Version from git import Repo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e9afb1f..8c4bcf91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,13 +94,13 @@ jobs: run_docs: ${{ needs.check-changes.outputs.run_docs }} lint: - needs: - - "check-changes" - if: needs.check-changes.outputs.run_tests == '1' uses: "./.github/workflows/lint.yml" build: - needs: "lint" + needs: + - "check-changes" + - "lint" + if: needs.check-changes.outputs.run_tests == '1' uses: "./.github/workflows/build.yml" test: diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 87a53dae..08594e38 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -95,7 +95,7 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": ""}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}}\ pulp_scenario_env: {}\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" @@ -109,7 +109,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": ""}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 7f69aa9b..1adcfc74 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -4,7 +4,6 @@ from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ObjectDoesNotExist from django.db import models -from django.conf import settings from pulpcore.plugin.models import ( Content, Publication, @@ -13,6 +12,7 @@ Repository, ) from pulpcore.plugin.responses import ArtifactResponse +from pulpcore.plugin.util import get_domain from pathlib import PurePath from .utils import ( @@ -77,7 +77,7 @@ def content_handler(self, path): elif len(path.parts) and path.parts[0] == "simple": # Temporary fix for PublishedMetadata not being properly served from remote storage # https://github.com/pulp/pulp_python/issues/413 - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if get_domain().storage_class != "pulpcore.app.models.storage.FileSystem": if self.publication or self.repository: try: publication = self.publication or Publication.objects.filter( diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index eb1082de..db6acdd0 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -3,7 +3,6 @@ import tempfile import json from collections import defaultdict -from django.core.files.storage import default_storage as storage from django.conf import settings from jinja2 import Template from packaging.utils import canonicalize_name @@ -144,8 +143,8 @@ def get_project_metadata_from_artifact(filename, artifact): # because pkginfo validates that the filename has a valid extension before # reading it with tempfile.NamedTemporaryFile('wb', dir=".", suffix=filename) as temp_file: - artifact_file = storage.open(artifact.file.name) - shutil.copyfileobj(artifact_file, temp_file) + artifact.file.seek(0) + shutil.copyfileobj(artifact.file, temp_file) temp_file.flush() metadata = DIST_TYPES[packagetype](temp_file.name) metadata.packagetype = packagetype diff --git a/template_config.yml b/template_config.yml index 33c3d396..80335113 100644 --- a/template_config.yml +++ b/template_config.yml @@ -2,18 +2,21 @@ # were not present before running plugin-template have been added with their default values. # generated with plugin_template +# +# After editing this file please always reapply the plugin template before committing any changes. -api_root: /pulp/ +--- +api_root: "/pulp/" black: false check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true -ci_base_image: ghcr.io/pulp/pulp-ci-centos9 +ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" ci_env: {} -ci_trigger: '{pull_request: {branches: [''*'']}}' -cli_package: pulp-cli -cli_repo: https://github.com/pulp/pulp-cli.git +ci_trigger: "{pull_request: {branches: ['*']}}" +cli_package: "pulp-cli" +cli_repo: "https://github.com/pulp/pulp-cli.git" core_import_allowed: [] deploy_client_to_pypi: true deploy_client_to_rubygems: true @@ -23,54 +26,62 @@ docker_fixtures: false extra_files: [] flake8: true flake8_ignore: [] -github_org: pulp +github_org: "pulp" latest_release_branch: null lint_requirements: true os_required_packages: [] parallel_test_workers: 8 -plugin_app_label: python -plugin_default_branch: main -plugin_name: pulp_python +plugin_app_label: "python" +plugin_default_branch: "main" +plugin_name: "pulp_python" plugins: -- app_label: python - name: pulp_python -post_job_template: null -pre_job_template: null + - app_label: "python" + name: "pulp_python" pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} pulp_env_s3: {} -pulp_scheme: https +pulp_scheme: "https" pulp_settings: - allowed_export_paths: /tmp - allowed_import_paths: /tmp + allowed_export_paths: "/tmp" + allowed_import_paths: "/tmp" orphan_protection_time: 0 - pypi_api_hostname: https://pulp:443 + pypi_api_hostname: "https://pulp:443" pulp_settings_azure: - AZURE_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== - AZURE_ACCOUNT_NAME: devstoreaccount1 - AZURE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; - AZURE_CONTAINER: pulp-test - AZURE_LOCATION: pulp3 - AZURE_OVERWRITE_FILES: true - AZURE_URL_EXPIRATION_SECS: 120 - DEFAULT_FILE_STORAGE: storages.backends.azure_storage.AzureStorage - MEDIA_ROOT: '' + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.azure_storage.AzureStorage" + OPTIONS: + account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + account_name: "devstoreaccount1" + azure_container: "pulp-test" + connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;" + expiration_secs: 120 + location: "pulp3" + overwrite_files: true + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" pulp_settings_gcp: null pulp_settings_s3: - AWS_ACCESS_KEY_ID: AKIAIT2Z5TDYPX3ARJBA - AWS_DEFAULT_ACL: '@none None' - AWS_S3_ADDRESSING_STYLE: path - AWS_S3_ENDPOINT_URL: http://minio:9000 - AWS_S3_REGION_NAME: eu-central-1 - AWS_S3_SIGNATURE_VERSION: s3v4 - AWS_SECRET_ACCESS_KEY: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS - AWS_STORAGE_BUCKET_NAME: pulp3 - DEFAULT_FILE_STORAGE: storages.backends.s3boto3.S3Boto3Storage - MEDIA_ROOT: '' + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.s3boto3.S3Boto3Storage" + OPTIONS: + access_key: "AKIAIT2Z5TDYPX3ARJBA" + addressing_style: "path" + bucket_name: "pulp3" + default_acl: "@none" + endpoint_url: "http://minio:9000" + region_name: "eu-central-1" + secret_key: "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS" + signature_version: "s3v4" + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" pydocstyle: true -release_email: pulp-infra@redhat.com -release_user: pulpbot +release_email: "pulp-infra@redhat.com" +release_user: "pulpbot" stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 @@ -86,4 +97,5 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true +... From d5c0a7d8d878fa462df9170e5da71ea119594242 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 26 Feb 2026 06:49:11 +0000 Subject: [PATCH 81/84] Update CI files --- .github/workflows/scripts/script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 0770527d..5a4910e3 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -144,7 +144,7 @@ else fi pushd ../pulp-cli pip install -r test_requirements.txt -pytest -v -m "pulp_python" +pytest -v tests -m "pulp_python" popd if [ -f "$POST_SCRIPT" ]; then From 61c40f4ed6a763d4dd3f32333a8945935e3f09e5 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 1 Mar 2026 03:28:56 +0000 Subject: [PATCH 82/84] Update CI files --- .ci/scripts/check_release.py | 25 ++-- .ci/scripts/clean_gh_release_notes.py | 33 +++++ .ci/scripts/collect_changes.py | 8 ++ .github/workflows/nightly.yml | 9 ++ .github/workflows/publish.yml | 2 +- .github/workflows/scripts/before_install.sh | 146 ++++++++++++++------ .github/workflows/scripts/before_script.sh | 52 ++++--- .github/workflows/scripts/install.sh | 122 +++------------- .github/workflows/test.yml | 5 +- 9 files changed, 218 insertions(+), 184 deletions(-) create mode 100755 .ci/scripts/clean_gh_release_notes.py diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 86e250e4..611c882c 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -11,7 +11,9 @@ import argparse import re import os +import sys import tomllib +import typing as t from pathlib import Path import yaml @@ -23,7 +25,7 @@ Z_CHANGELOG_EXTS = [".bugfix", ".misc"] -def options(): +def options() -> argparse.Namespace: """Check which branches need a release.""" parser = argparse.ArgumentParser() parser.add_argument( @@ -42,13 +44,13 @@ def options(): return parser.parse_args() -def template_config(): +def template_config() -> dict[str, t.Any]: # Assume this script lies in .ci/scripts path = Path(__file__).absolute().parent.parent.parent / "template_config.yml" return yaml.safe_load(path.read_text()) -def current_version(repo, commitish): +def current_version(repo: Repo, commitish: str) -> Version: try: pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml")) try: @@ -62,7 +64,7 @@ def current_version(repo, commitish): return Version(current_version) -def check_pyproject_dependencies(repo, from_commit, to_commit): +def check_pyproject_dependencies(repo: Repo, from_commit: str, to_commit: str) -> list[str]: try: new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml")) try: @@ -83,8 +85,8 @@ def check_pyproject_dependencies(repo, from_commit, to_commit): return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."] -def main(options, template_config): - DEFAULT_BRANCH = template_config["plugin_default_branch"] +def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int: + DEFAULT_BRANCH: str = template_config["plugin_default_branch"] repo = Repo() @@ -97,7 +99,7 @@ def main(options, template_config): # Warning: This will not work if branch names contain "/" but we don't really care here. heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")] - available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] + available_branches = [h for h in heads if re.fullmatch(RELEASE_BRANCH_REGEX, h)] available_branches.sort(key=lambda ver: Version(ver)) available_branches.append(DEFAULT_BRANCH) @@ -114,7 +116,10 @@ def main(options, template_config): if diff := branches - set(available_branches): print(f"Supplied branches contains non-existent branches! {diff}") - exit(1) + return 1 + + branches = [branch for branch in available_branches if branch in branches] + branches.reverse() print(f"Checking for releases on branches: {branches}") @@ -179,6 +184,8 @@ def main(options, template_config): if len(releases) == 0: print("No new releases to perform.") + return 0 + if __name__ == "__main__": - main(options(), template_config()) + sys.exit(main(options(), template_config())) diff --git a/.ci/scripts/clean_gh_release_notes.py b/.ci/scripts/clean_gh_release_notes.py new file mode 100755 index 00000000..2bf5f4c2 --- /dev/null +++ b/.ci/scripts/clean_gh_release_notes.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# This script is running with elevated privileges from the main branch against pull requests. +# +# It cleans the input from artifacts which are used by the pulp documentation internally, +# but clutter for GitHub releases + +import sys + +NOTE = """ +> [!NOTE] +> Official changes are available on [Pulp docs]({docs_url})\ +""" + + +def main(): + plugin_name = sys.argv[1] + version_str = sys.argv[2] + docs_url = f"https://pulpproject.org/{plugin_name}/changes/#{version_str}" + note_added = False + for line in sys.stdin: + if line.endswith("\n"): + line = line[:-1] + if line.startswith("#"): + print(line.split(" {: #")[0]) + if not note_added and version_str in line: + print(NOTE.format(docs_url=docs_url)) + note_added = True + else: + print(line) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 877ebc8c..fbb5d59d 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -1,4 +1,12 @@ #!/bin/env python3 +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# ] +# /// + # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ec3fcb11..949c8e29 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -61,6 +61,7 @@ jobs: - name: Create Pull Request uses: peter-evans/create-pull-request@v6 + id: "create_pr_changelog" with: token: ${{ secrets.RELEASE_TOKEN }} title: "Update Changelog" @@ -68,4 +69,12 @@ jobs: branch: "changelog/update" delete-branch: true path: "pulp_python" + - name: "Mark PR automerge" + working-directory: "pulp_python" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_changelog.outputs.pull-request-number }}" + if: "steps.create_pr_changelog.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true ... diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 525144fa..feba4df9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -136,7 +136,7 @@ jobs: run: | # The last commit before the release commit contains the release CHANGES fragments git checkout "${TAG_NAME}~" - NOTES=$(towncrier build --draft --version $TAG_NAME) + NOTES=$(towncrier build --draft --version $TAG_NAME | .ci/scripts/clean_gh_release_notes.py pulp_python $TAG_NAME) echo "body<> $GITHUB_OUTPUT echo "$NOTES" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index c31ec734..30c3ff18 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -7,66 +7,124 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script prepares the scenario definition in the .ci/ansible/vars/main.yaml file. +# +# It requires the following environment: +# TEST - The name of the scenario to prepare. +# +# It may also dump the {lower,upper}bounds_constraints.txt for the specific scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -mveuo pipefail +if [ -f .github/workflows/scripts/pre_before_install.sh ]; then + source .github/workflows/scripts/pre_before_install.sh +fi -if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ] -then - BRANCH_BUILD=0 -else - BRANCH_BUILD=1 - BRANCH="${GITHUB_REF##refs/heads/}" +COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" +COMPONENT_SOURCE="./pulp_python/dist/pulp_python-${COMPONENT_VERSION}-py3-none-any.whl" +if [ "$TEST" = "s3" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3]" fi -if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ] -then - TAG_BUILD=0 -else - TAG_BUILD=1 - BRANCH="${GITHUB_REF##refs/tags/}" +if [ "$TEST" = "azure" ]; then + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]" fi -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export COMMIT_MSG +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt +fi +if [[ "$TEST" = "lowerbounds" ]]; then + python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt +fi +export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") -COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) +echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" -mkdir .ci/ansible/vars || true -echo "---" > .ci/ansible/vars/main.yaml -echo "legacy_component_name: pulp_python" >> .ci/ansible/vars/main.yaml -echo "component_name: python" >> .ci/ansible/vars/main.yaml -echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml +# Compose the scenario definition. +mkdir -p .ci/ansible/vars -export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh -export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh +cat > .ci/ansible/vars/main.yaml << VARSYAML +--- +scenario: "${TEST}" +legacy_component_name: "pulp_python" +component_name: "python" +component_version: "${COMPONENT_VERSION}" +pulp_env: {} +pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} +pulp_scheme: "https" +pulp_default_container: "ghcr.io/pulp/pulp-ci-centos9:latest" +api_root: "${PULP_API_ROOT}" +image: + name: "pulp" + tag: "ci_build" +plugins: + - name: "pulp_python" + source: "${COMPONENT_SOURCE}" + ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) + upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) + lowerounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) +services: + - name: "pulp" + image: "pulp:ci_build" + volumes: + - "./settings:/etc/pulp" + - "./ssh:/keys/" + - "~/.config:/var/lib/pulp/.config" + - "../../../pulp-openapi-generator:/root/pulp-openapi-generator" + env: + PULP_WORKERS: "4" + PULP_HTTPS: "true" +VARSYAML -if [ -f $PRE_BEFORE_INSTALL ]; then - source $PRE_BEFORE_INSTALL +if [ "$TEST" = "s3" ]; then + MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA + MINIO_SECRET_KEY=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 + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "minio" + image: "minio/minio" + env: + MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}" + MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}" + command: "server /data" +s3_test: true +minio_access_key: "${MINIO_ACCESS_KEY}" +minio_secret_key: "${MINIO_SECRET_KEY}" +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}} +pulp_scenario_env: {} +VARSYAML fi -if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] -then - echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml +if [ "$TEST" = "azure" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-azurite" + image: "mcr.microsoft.com/azure-storage/azurite" + command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0" +azure_test: true +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}} +pulp_scenario_env: {} +VARSYAML fi -for i in {1..3} -do - ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 -done -if [[ $s -gt 0 ]] -then - echo "Failed to install amazon.aws" - exit $s +if [ "$TEST" = "gcp" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "ci-gcp" + image: "fsouza/fake-gcs-server" + volumes: + - "storage_data:/etc/pulp" + command: " -scheme http" +gcp_test: true +pulp_scenario_settings: null +pulp_scenario_env: {} +VARSYAML fi -if [[ "$TEST" = "pulp" ]]; then - python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt -fi -if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt -fi +cat >> .ci/ansible/vars/main.yaml << VARSYAML +... +VARSYAML +cat .ci/ansible/vars/main.yaml -if [ -f $POST_BEFORE_INSTALL ]; then - source $POST_BEFORE_INSTALL +if [ -f .github/workflows/scripts/post_before_install.sh ]; then + source .github/workflows/scripts/post_before_install.sh fi diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 11404f45..6af7098e 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -7,48 +7,46 @@ # # For more info visit https://github.com/pulp/plugin_template +# This script dumps some files to help understand the setup of the test scenario. + +set -eu -o pipefail + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -euv - source .github/workflows/scripts/utils.sh -export PRE_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/pre_before_script.sh -export POST_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/post_before_script.sh - -if [[ -f $PRE_BEFORE_SCRIPT ]]; then - source $PRE_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/pre_before_script.sh ]]; then + source .github/workflows/scripts/pre_before_script.sh fi -# Developers should be able to reproduce the containers with this config -echo "CI vars:" -tail -v -n +1 .ci/ansible/vars/main.yaml - # Developers often want to know the final pulp config -echo "PULP CONFIG:" +echo +echo "# Pulp config:" tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json -echo "Containerfile:" +echo +echo "# Containerfile:" tail -v -n +1 .ci/ansible/Containerfile -echo "Constraints Files:" -# The need not even exist. +echo +echo "# Constraints Files:" +# They need not even exist. tail -v -n +1 ../*/*constraints.txt || true -# Needed for some functional tests -cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" -cmd_prefix bash -c "usermod -a -G wheel pulp" +echo +echo "# pip list outside the container" +pip list -if [[ "${REDIS_DISABLED:-false}" == true ]]; then - cmd_prefix bash -c "s6-rc -d change redis" - echo "The Redis service was disabled for $TEST" -fi +echo +echo "# pip list inside the container" +cmd_prefix bash -c "pip3 list" + +echo +echo "# State of the containers" +docker ps -a -if [[ -f $POST_BEFORE_SCRIPT ]]; then - source $POST_BEFORE_SCRIPT +if [[ -f .github/workflows/scripts/post_before_script.sh ]]; then + source .github/workflows/scripts/post_before_script.sh fi -# Lots of plugins try to use this path, and throw warnings if they cannot access it. -cmd_prefix mkdir /.pytest_cache -cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 08594e38..4947df63 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -7,120 +7,38 @@ # # For more info visit https://github.com/pulp/plugin_template +set -euv + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. REPO_ROOT="$PWD" -set -euv - source .github/workflows/scripts/utils.sh -PLUGIN_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" -PLUGIN_SOURCE="./pulp_python/dist/pulp_python-${PLUGIN_VERSION}-py3-none-any.whl" - -export PULP_API_ROOT="/pulp/" - PIP_REQUIREMENTS=("pulp-cli") # This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} +if [[ "$TEST" = "s3" ]]; then +for i in {1..3} +do + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 +done +if [[ $s -gt 0 ]] +then + echo "Failed to install amazon.aws" + exit $s +fi +fi + # Check out the pulp-cli branch matching the installed version. PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli cd .ci/ansible/ -if [ "$TEST" = "s3" ]; then - PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[s3]" -fi -if [ "$TEST" = "azure" ]; then - PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[azure]" -fi - -cat >> vars/main.yaml << VARSYAML -image: - name: pulp - tag: "ci_build" -plugins: - - name: pulp_python - source: "${PLUGIN_SOURCE}" -VARSYAML -if [[ -f ../../ci_requirements.txt ]]; then - cat >> vars/main.yaml << VARSYAML - ci_requirements: true -VARSYAML -fi -if [ "$TEST" = "pulp" ]; then - cat >> vars/main.yaml << VARSYAML - upperbounds: true -VARSYAML -fi -if [ "$TEST" = "lowerbounds" ]; then - cat >> vars/main.yaml << VARSYAML - lowerbounds: true -VARSYAML -fi - -cat >> vars/main.yaml << VARSYAML -services: - - name: pulp - image: "pulp:ci_build" - volumes: - - ./settings:/etc/pulp - - ./ssh:/keys/ - - ~/.config:/var/lib/pulp/.config - - ../../../pulp-openapi-generator:/root/pulp-openapi-generator - env: - PULP_WORKERS: "4" - PULP_HTTPS: "true" -VARSYAML - -cat >> vars/main.yaml << VARSYAML -pulp_env: {} -pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} -pulp_scheme: https -pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest -VARSYAML - -if [ "$TEST" = "s3" ]; then - export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA - export MINIO_SECRET_KEY=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 - sed -i -e '/^services:/a \ - - name: minio\ - image: minio/minio\ - env:\ - MINIO_ACCESS_KEY: "'$MINIO_ACCESS_KEY'"\ - MINIO_SECRET_KEY: "'$MINIO_SECRET_KEY'"\ - command: "server /data"' vars/main.yaml - sed -i -e '$a s3_test: true\ -minio_access_key: "'$MINIO_ACCESS_KEY'"\ -minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}}\ -pulp_scenario_env: {}\ -' vars/main.yaml - export PULP_API_ROOT="/rerouted/djnd/" -fi - -if [ "$TEST" = "azure" ]; then - sed -i -e '/^services:/a \ - - name: ci-azurite\ - image: mcr.microsoft.com/azure-storage/azurite\ - volumes:\ - - ./azurite:/etc/pulp\ - command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml - sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}}\ -pulp_scenario_env: {}\ -' vars/main.yaml -fi - -echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" - -if [ "${PULP_API_ROOT:-}" ]; then - sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml -fi -pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" --username "admin" --password "password" +pulp config create --base-url https://pulp --api-root "${PULP_API_ROOT}" --username "admin" --password "password" cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml @@ -155,6 +73,10 @@ if [[ "$TEST" = "azure" ]]; then az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING fi -echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list" -echo ::endgroup:: +# Needed for some functional tests +cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" +cmd_prefix bash -c "usermod -a -G wheel pulp" + +# Lots of plugins try to use this path, and throw warnings if they cannot access it. +cmd_prefix mkdir /.pytest_cache +cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88cfd8f3..4e167808 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,7 +79,7 @@ jobs: run: | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - name: "Before Install" + - name: "Prepare Scenario Definition" run: | .github/workflows/scripts/before_install.sh shell: "bash" @@ -99,7 +99,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - name: "Before Script" + - name: "Dump CI Metadata" run: | .github/workflows/scripts/before_script.sh shell: "bash" @@ -108,7 +108,6 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" - name: "Script" run: | From 153a4d5cec37aad47bda868ba3144762feee5d4a Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 8 Mar 2026 03:21:02 +0000 Subject: [PATCH 83/84] Update CI files --- .github/workflows/scripts/before_install.sh | 2 +- pyproject.toml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 30c3ff18..39704c86 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -64,7 +64,7 @@ plugins: source: "${COMPONENT_SOURCE}" ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) - lowerounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) + lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) services: - name: "pulp" image: "pulp:ci_build" diff --git a/pyproject.toml b/pyproject.toml index ec98c6e9..d30d6c21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ ignore = [ ".ci/**", "lint_requirements.txt", ".flake8", + "AGENTS.md", + "CLAUDE.md", ] [tool.bumpversion] From 1a1f2b50b09b77e8c49711fc549678a4ac944cba Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 15 Mar 2026 03:34:11 +0000 Subject: [PATCH 84/84] Update CI files --- .ci/ansible/Containerfile.j2 | 43 ++++++-------- .ci/ansible/build_container.yaml | 11 ++-- .ci/ansible/start_container.yaml | 66 +++++++++++---------- .github/workflows/build.yml | 18 +++--- .github/workflows/ci.yml | 37 ++++++------ .github/workflows/codeql-analysis.yml | 33 ++++++----- .github/workflows/create-branch.yml | 57 +++++++++--------- .github/workflows/docs.yml | 15 +++-- .github/workflows/lint.yml | 32 +++------- .github/workflows/nightly.yml | 21 ++++--- .github/workflows/pr_checks.yml | 6 +- .github/workflows/publish.yml | 17 +++--- .github/workflows/release.yml | 7 +-- .github/workflows/sanity.yml | 54 +++++++++++++++++ .github/workflows/scripts/before_install.sh | 22 +++---- .github/workflows/scripts/before_script.sh | 2 +- .github/workflows/scripts/install.sh | 4 -- .github/workflows/test.yml | 20 +++---- .github/workflows/update-labels.yml | 7 +-- .github/workflows/update_ci.yml | 12 ++-- template_config.yml | 1 - 21 files changed, 253 insertions(+), 232 deletions(-) create mode 100644 .github/workflows/sanity.yml diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 01469b64..fb9ed4a1 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,34 +1,28 @@ -FROM {{ ci_base | default(pulp_default_container) }} +FROM {{ image.ci_base }} +{%- if image.webserver_snippet %} -# Add source directories to container -{% for item in plugins %} -ADD ./{{ item.name }} ./{{ item.name }} -{% endfor %} +ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf +{%- endif %} + +{%- for item in extra_files | default([]) %} -{% for item in extra_files | default([]) %} ADD ./{{ item.origin }} {{ item.destination }} -{% endfor %} +{%- endfor %} # This MUST be the ONLY call to pip install in inside the container. RUN pip3 install --upgrade pip setuptools wheel && \ rm -rf /root/.cache/pip && \ - pip3 install -{%- if s3_test | default(false) -%} -{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue -{%- endif -%} -{%- for item in plugins -%} -{{ " " }}{{ item.source }} -{%- if item.upperbounds | default(false) -%} -{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt + pip3 install {{ image.source }} +{%- if image.upperbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt {%- endif -%} -{%- if item.lowerbounds | default(false) -%} -{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt +{%- if image.lowerbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt {%- endif -%} -{%- if item.ci_requirements | default(false) -%} -{{ " " }}-r ./{{ item.name }}/ci_requirements.txt +{%- if image.ci_requirements | default(false) -%} +{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt {%- endif -%} -{%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ +{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \ rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} @@ -46,11 +40,8 @@ ENV {{ key | upper }}={{ value }} USER pulp:pulp RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link -USER root:root -{% for item in plugins %} -RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ - ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true -{% endfor %} +RUN mkdir /var/lib/pulp/.config +USER root:root ENTRYPOINT ["/init"] diff --git a/.ci/ansible/build_container.yaml b/.ci/ansible/build_container.yaml index c380b430..0a188cba 100644 --- a/.ci/ansible/build_container.yaml +++ b/.ci/ansible/build_container.yaml @@ -1,15 +1,14 @@ # Ansible playbook to create the pulp service containers image --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Generate Containerfile from template" - template: - src: Containerfile.j2 - dest: Containerfile - + ansible.builtin.template: + src: "Containerfile.j2" + dest: "Containerfile" - name: "Build pulp image" # We build from the ../.. (parent dir of pulpcore git repo) Docker build # "context" so that repos like pulp-smash are accessible to Docker diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 47e5221e..acdc22ad 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -1,33 +1,26 @@ # Ansible playbook to start the pulp service container and its supporting services --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Create Settings Directories" - file: + ansible.builtin.file: path: "{{ item }}" - state: directory + state: "directory" mode: "0755" loop: - - settings - - ssh - - ~/.config/pulp_smash + - "settings" - name: "Generate Pulp Settings" template: - src: settings.py.j2 - dest: settings/settings.py - - - name: "Configure pulp-smash" - copy: - src: smash-config.json - dest: ~/.config/pulp_smash/settings.json + src: "settings.py.j2" + dest: "settings/settings.py" - name: "Setup docker networking" docker_network: - name: pulp_ci_bridge + name: "pulp_ci_bridge" - name: "Start Service Containers" docker_container: @@ -37,24 +30,24 @@ recreate: true privileged: true networks: - - name: pulp_ci_bridge + - name: "pulp_ci_bridge" aliases: "{{ item.name }}" volumes: "{{ item.volumes | default(omit) }}" env: "{{ item.env | default(omit) }}" command: "{{ item.command | default(omit) }}" - state: started + state: "started" loop: "{{ services | default([]) }}" - name: "Retrieve Docker Network Info" docker_network_info: - name: pulp_ci_bridge - register: pulp_ci_bridge_info + name: "pulp_ci_bridge" + register: "pulp_ci_bridge_info" - name: "Update /etc/hosts" lineinfile: - path: /etc/hosts + path: "/etc/hosts" regexp: "\\s{{ item.value.Name }}\\s*$" - line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}" + line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}" loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}" become: true @@ -63,19 +56,19 @@ aws_access_key: "{{ minio_access_key }}" aws_secret_key: "{{ minio_secret_key }}" s3_url: "http://minio:9000" - region: eu-central-1 - name: pulp3 - state: present - when: s3_test | default(false) + region: "eu-central-1" + name: "pulp3" + state: "present" + when: "s3_test | default(false)" - block: - name: "Wait for Pulp" uri: url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/" - follow_redirects: all - validate_certs: no - register: result - until: result.status == 200 + follow_redirects: "all" + validate_certs: "no" + register: "result" + until: "result.status == 200" retries: 12 delay: 5 rescue: @@ -86,7 +79,7 @@ - name: "Check version of component being tested" assert: that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver) + - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)" fail_msg: | Component {{ item.app_label }} was expected to be installed in version {{ component_version }}. Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}. @@ -100,9 +93,20 @@ login admin password password -- hosts: pulp +- hosts: "pulp" gather_facts: false tasks: + - name: "Create directory for pulp-smash config" + ansible.builtin.file: + path: "/var/lib/pulp/.config/pulp_smash/" + state: "directory" + mode: "0755" + + - name: "Configure pulp-smash" + ansible.builtin.copy: + src: "smash-config.json" + dest: "/var/lib/pulp/.config/pulp_smash/settings.json" + - name: "Set pulp admin password" command: cmd: "pulpcore-manager reset-admin-password --password password" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 67878dc4..988902a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,23 +19,21 @@ jobs: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install build packaging twine wheel mkdocs jq - echo ::endgroup:: - name: "Build package" run: | python3 -m build @@ -48,7 +46,7 @@ jobs: pulpcore-manager openapi --file "api.json" pulpcore-manager openapi --bindings --component "python" --file "python-api.json" - name: "Upload Package whl" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "plugin_package" path: "pulp_python/dist/" @@ -56,7 +54,7 @@ jobs: retention-days: 5 overwrite: true - name: "Upload API specs" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "api_spec" path: | @@ -75,7 +73,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - name: "Upload python client packages" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "python-client.tar" path: | @@ -84,7 +82,7 @@ jobs: retention-days: 5 overwrite: true - name: "Upload python client docs" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "python-client-docs.tar" path: | @@ -102,7 +100,7 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - name: "Upload Ruby client" - uses: "actions/upload-artifact@v4" + uses: "actions/upload-artifact@v5" with: name: "ruby-client.tar" path: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c4bcf91..2965afca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,18 +21,16 @@ jobs: check-commits: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install requests pygithub pyyaml - echo ::endgroup:: - name: "Check commit message" if: github.event_name == 'pull_request' env: @@ -44,29 +42,27 @@ jobs: .github/workflows/scripts/check_commit.sh check-changes: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" outputs: - run_tests: ${{ steps.check.outputs.run_tests }} - run_docs: ${{ steps.check.outputs.run_docs }} + run_tests: "${{ steps.check.outputs.run_tests }}" + run_docs: "${{ steps.check.outputs.run_docs }}" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.12" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install gitpython - echo ::endgroup:: - - name: Analyze changed files - shell: bash - id: check + - name: "Analyze changed files" + id: "check" + shell: "bash" run: | # We only test docs on the default branch (usually main) if [[ "${{ github.base_ref }}" == *"main" ]]; then @@ -91,16 +87,19 @@ jobs: needs: "check-changes" uses: "./.github/workflows/docs.yml" with: - run_docs: ${{ needs.check-changes.outputs.run_docs }} + run_docs: "${{ needs.check-changes.outputs.run_docs }}" lint: uses: "./.github/workflows/lint.yml" + sanity: + uses: "./.github/workflows/sanity.yml" + build: needs: - "check-changes" - "lint" - if: needs.check-changes.outputs.run_tests == '1' + if: "needs.check-changes.outputs.run_tests == '1'" uses: "./.github/workflows/build.yml" test: @@ -112,7 +111,7 @@ jobs: deprecations: runs-on: "ubuntu-latest" - if: github.base_ref == 'main' + if: "github.base_ref == 'main'" needs: "test" steps: - name: "Create working directory" @@ -120,7 +119,7 @@ jobs: mkdir -p "pulp_python" working-directory: "." - name: "Download Deprecations" - uses: actions/download-artifact@v4 + uses: "actions/download-artifact@v8" with: pattern: "deprecations-*" path: "pulp_python" @@ -139,6 +138,7 @@ jobs: - "lint" - "test" - "docs" + - "sanity" if: "always()" steps: - name: "Collect needed jobs results" @@ -171,3 +171,4 @@ jobs: check_jobs "$FILTERS" echo "CI says: Looks good!" +... diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 899147c4..22e5d99b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,39 +4,42 @@ # './plugin-template --github pulp_python' to update this file. # # For more info visit https://github.com/pulp/plugin_template +--- name: "Python CodeQL" on: workflow_dispatch: schedule: - - cron: '37 1 * * 6' + - cron: "37 1 * * 6" concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: analyze: - name: Analyze - runs-on: ubuntu-latest + name: "Analyze" + runs-on: "ubuntu-latest" permissions: - actions: read - contents: read - security-events: write + actions: "read" + contents: "read" + security-events: "write" strategy: fail-fast: false matrix: - language: [ 'python' ] + language: + - "python" steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: "Checkout repository" + uses: "actions/checkout@v6" - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + - name: "Initialize CodeQL" + uses: "github/codeql-action/init@v4" with: - languages: ${{ matrix.language }} + languages: "${{ matrix.language }}" - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: "Perform CodeQL Analysis" + uses: "github/codeql-action/analyze@v4" +... diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 75c3224d..6b57c785 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Create New Release Branch +name: "Create New Release Branch" on: workflow_dispatch: @@ -15,35 +15,33 @@ env: jobs: create-branch: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false permissions: - contents: write + contents: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install bump-my-version packaging -r plugin_template/requirements.txt - echo ::endgroup:: - name: "Setting secrets" working-directory: "pulp_python" @@ -52,8 +50,8 @@ jobs: env: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: Determine new branch name - working-directory: pulp_python + - name: "Determine new branch name" + working-directory: "pulp_python" run: | # Just to be sure... git checkout main @@ -65,42 +63,43 @@ jobs: fi echo "NEW_BRANCH=${NEW_BRANCH}" >> "$GITHUB_ENV" - - name: Create release branch - working-directory: pulp_python + - name: "Create release branch" + working-directory: "pulp_python" run: | git branch "${NEW_BRANCH}" - - name: Bump version on main branch - working-directory: pulp_python + - name: "Bump version on main branch" + working-directory: "pulp_python" run: | bump-my-version bump --no-commit minor - - name: Remove entries from CHANGES directory - working-directory: pulp_python + - name: "Remove entries from CHANGES directory" + working-directory: "pulp_python" run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + - - name: Update CI branches in template_config - working-directory: plugin_template + - name: "Update CI branches in template_config" + working-directory: "plugin_template" run: | python3 ./plugin-template pulp_python --github --latest-release-branch "${NEW_BRANCH}" git add -A - - name: Make a PR with version bump and without CHANGES/* - uses: peter-evans/create-pull-request@v6 + - name: "Make a PR with version bump and without CHANGES/*" + uses: "peter-evans/create-pull-request@v8" with: - path: pulp_python - token: ${{ secrets.RELEASE_TOKEN }} - committer: pulpbot - author: pulpbot - branch: minor-version-bump - base: main - title: Bump minor version + path: "pulp_python" + token: "${{ secrets.RELEASE_TOKEN }}" + committer: "pulpbot " + author: "pulpbot " + branch: "minor-version-bump" + base: "main" + title: "Bump minor version" commit-message: | Bump minor version delete-branch: true - - name: Push release branch - working-directory: pulp_python + - name: "Push release branch" + working-directory: "pulp_python" run: | git push origin "${NEW_BRANCH}" +... diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9e16317d..c897fcac 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,7 +13,7 @@ on: run_docs: description: "Whether to run docs jobs" required: true - type: string + type: "string" jobs: changelog: @@ -22,23 +22,22 @@ jobs: run: working-directory: "pulp_python" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.12" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install towncrier - echo ::endgroup:: - name: "Build changelog" run: | towncrier build --yes --version 4.0.0.ci docs: - if: ${{ inputs.run_docs == '1' }} - uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@main' + if: "${{ inputs.run_docs == '1' }}" + uses: "pulp/pulp-docs/.github/workflows/docs-ci.yml@main" with: - pulpdocs_ref: 'main' + pulpdocs_ref: "main" +... diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9dddcbdd..d07a39a0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,51 +19,33 @@ jobs: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install -r lint_requirements.txt - echo ::endgroup:: - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows - - name: "Verify bump version config" - run: | - bump-my-version bump --dry-run release - bump-my-version show-bump - # Lint code. - name: "Run flake8" run: | flake8 - - name: "Run extra lint checks" - run: | - [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - - - name: "Check for any files unintentionally left out of MANIFEST.in" - run: | - check-manifest - - - name: "Verify requirements files" - run: | - python .ci/scripts/check_requirements.py - - - name: "Check for pulpcore imports outside of pulpcore.plugin" - run: | - sh .ci/scripts/check_pulpcore_imports.sh - - name: "Check for common gettext problems" run: | sh .ci/scripts/check_gettext.sh + + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh +... diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 949c8e29..ed8149ce 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ on: schedule: # * is a special character in YAML so you have to quote this string # runs at 3:00 UTC daily - - cron: '00 3 * * *' + - cron: "00 3 * * *" workflow_dispatch: defaults: @@ -34,36 +34,35 @@ jobs: [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] changelog: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.13" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install gitpython packaging toml - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Collect changes from all branches - run: python .ci/scripts/collect_changes.py + - name: "Collect changes from all branches" + run: | + python .ci/scripts/collect_changes.py - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 + - name: "Create Pull Request" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_changelog" with: - token: ${{ secrets.RELEASE_TOKEN }} + token: "${{ secrets.RELEASE_TOKEN }}" title: "Update Changelog" body: "" branch: "changelog/update" diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 0e0a7936..d8054896 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -32,10 +32,10 @@ jobs: permissions: pull-requests: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Determine PR labels" @@ -43,7 +43,7 @@ jobs: pip install GitPython==3.1.42 git fetch origin ${{ github.event.pull_request.head.sha }} python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" - - uses: "actions/github-script@v7" + - uses: "actions/github-script@v8" name: "Apply PR Labels" with: script: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index feba4df9..6d0d3aa8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -47,7 +47,7 @@ jobs: id-token: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" @@ -63,7 +63,7 @@ jobs: tar -xvf python-python-client.tar - name: "Publish client to pypi" - uses: pypa/gh-action-pypi-publish@release/v1 + uses: "pypa/gh-action-pypi-publish@release/v1" with: packages-dir: "pulp_python/dist/" publish-ruby-bindings: @@ -76,7 +76,7 @@ jobs: id-token: "write" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" @@ -111,18 +111,18 @@ jobs: - "publish-ruby-bindings" permissions: - contents: write + contents: "write" env: TAG_NAME: "${{ github.ref_name }}" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" @@ -142,9 +142,9 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - name: "Create release on GitHub" - uses: "actions/github-script@v7" + uses: "actions/github-script@v8" env: - RELEASE_BODY: ${{ steps.get_release_notes.outputs.body }} + RELEASE_BODY: "${{ steps.get_release_notes.outputs.body }}" with: script: | const { TAG_NAME, RELEASE_BODY } = process.env; @@ -156,3 +156,4 @@ jobs: body: RELEASE_BODY, make_latest: "legacy", }); +... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b758bae6..e056bcfa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,21 +22,19 @@ jobs: fail-fast: false steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" token: ${{ secrets.RELEASE_TOKEN }} - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install bump-my-version towncrier - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | @@ -58,3 +56,4 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" +... diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml new file mode 100644 index 00000000..223f07cb --- /dev/null +++ b/.github/workflows/sanity.yml @@ -0,0 +1,54 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + + +# This file describes checks that should prevent a premature merge, +# but still let the tests run for demonstrations or experiments. +--- +name: "Sanity" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + sanity: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v6" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + pip install -r lint_requirements.txt + + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest + + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py + + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh +... diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 39704c86..782c6ae8 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -26,7 +26,7 @@ fi COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" COMPONENT_SOURCE="./pulp_python/dist/pulp_python-${COMPONENT_VERSION}-py3-none-any.whl" if [ "$TEST" = "s3" ]; then - COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3]" + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" fi if [ "$TEST" = "azure" ]; then COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]" @@ -38,6 +38,7 @@ fi if [[ "$TEST" = "lowerbounds" ]]; then python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi + export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" @@ -48,30 +49,31 @@ mkdir -p .ci/ansible/vars cat > .ci/ansible/vars/main.yaml << VARSYAML --- scenario: "${TEST}" +plugin_name: "pulp_python" legacy_component_name: "pulp_python" component_name: "python" component_version: "${COMPONENT_VERSION}" pulp_env: {} pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} pulp_scheme: "https" -pulp_default_container: "ghcr.io/pulp/pulp-ci-centos9:latest" api_root: "${PULP_API_ROOT}" image: name: "pulp" tag: "ci_build" -plugins: - - name: "pulp_python" - source: "${COMPONENT_SOURCE}" - ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) - upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) - lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) + ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest" + source: "${COMPONENT_SOURCE}" + ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) + upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) + lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) + webserver_snippet: $(test -f pulp_python/app/webserver_snippets/nginx.conf && echo -n true || echo -n false ) +extra_files: + - origin: "pulp_python" + destination: "pulp_python" services: - name: "pulp" image: "pulp:ci_build" volumes: - "./settings:/etc/pulp" - - "./ssh:/keys/" - - "~/.config:/var/lib/pulp/.config" - "../../../pulp-openapi-generator:/root/pulp-openapi-generator" env: PULP_WORKERS: "4" diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 6af7098e..13809a87 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -23,7 +23,7 @@ fi # Developers often want to know the final pulp config echo echo "# Pulp config:" -tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json +tail -v -n +1 .ci/ansible/settings/settings.* echo echo "# Containerfile:" diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 4947df63..9df47ccd 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -44,10 +44,6 @@ cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml ansible-playbook start_container.yaml -# .config needs to be accessible by the pulp user in the container, but some -# files will likely be modified on the host by post/pre scripts. -chmod 777 ~/.config/pulp_smash/ -chmod 666 ~/.config/pulp_smash/settings.json # Plugins often write to ~/.config/pulp/cli.toml from the host chmod 777 ~/.config/pulp chmod 666 ~/.config/pulp/cli.toml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e167808..fbddb788 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ on: inputs: matrix_env: required: true - type: string + type: "string" defaults: run: @@ -24,38 +24,38 @@ jobs: strategy: fail-fast: false matrix: - env: ${{ fromJSON(inputs.matrix_env) }} + env: "${{ fromJSON(inputs.matrix_env) }}" steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 path: "pulp_python" - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 1 repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Download plugin package" - uses: "actions/download-artifact@v4" + uses: "actions/download-artifact@v8" with: name: "plugin_package" path: "pulp_python/dist/" - name: "Download API specs" - uses: "actions/download-artifact@v4" + uses: "actions/download-artifact@v8" with: name: "api_spec" path: "pulp_python/" - name: "Download client packages" - uses: "actions/download-artifact@v4" + uses: "actions/download-artifact@v8" with: name: "python-client.tar" path: "pulp_python" @@ -70,10 +70,8 @@ jobs: - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install build towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - name: "Set environment variables" run: | @@ -124,7 +122,7 @@ jobs: docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt - name: "Upload Deprecations" - uses: actions/upload-artifact@v4 + uses: "actions/upload-artifact@v5" with: name: "deprecations-${{ matrix.env.TEST }}" path: "pulp_python/deprecations-${{ matrix.env.TEST }}.txt" diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index 4565da8c..39320dca 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -19,7 +19,7 @@ jobs: update_backport_labels: runs-on: "ubuntu-latest" steps: - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Configure Git with pulpbot name and email" @@ -28,12 +28,11 @@ jobs: git config --global user.email 'pulp-infra@redhat.com' - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install requests pyyaml - echo ::endgroup:: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" - name: "Update labels" run: | python3 .github/workflows/scripts/update_backport_labels.py env: GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" +... diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 05b93764..0c6c2f8f 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -12,7 +12,7 @@ on: schedule: # * is a special character in YAML so you have to quote this string # runs at 2:30 UTC every Sunday - - cron: '30 2 * * 0' + - cron: "30 2 * * 0" workflow_dispatch: jobs: @@ -23,27 +23,25 @@ jobs: fail-fast: false steps: - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v5" + - uses: "actions/setup-python@v6" with: python-version: "3.11" - name: "Install python dependencies" run: | - echo ::group::PYDEPS pip install gitpython packaging -r plugin_template/requirements.txt - echo ::endgroup:: - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - uses: "actions/checkout@v4" + - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "pulp_python" @@ -55,7 +53,7 @@ jobs: ../plugin_template/scripts/update_ci.sh - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" + uses: "peter-evans/create-pull-request@v8" id: "create_pr_main" with: token: "${{ secrets.RELEASE_TOKEN }}" diff --git a/template_config.yml b/template_config.yml index 80335113..4f12934a 100644 --- a/template_config.yml +++ b/template_config.yml @@ -94,7 +94,6 @@ test_deprecations: true test_gcp: false test_lowerbounds: true test_performance: false -test_reroute: true test_s3: true use_issue_template: true ...