From 4d4a3839fafb00db03638553a9d93f10adc8f14c Mon Sep 17 00:00:00 2001 From: Benjamin Renard Date: Mon, 22 Jan 2024 00:20:26 +0100 Subject: [PATCH] Introduce pre-commit hooks and code cleaning --- .pre-commit-config.yaml | 67 ++++++++++++ README.md | 46 +++++---- aptly-publish | 223 +++++++++++++++++----------------------- docs.md | 46 ++++----- 4 files changed, 211 insertions(+), 171 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c710eb1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,67 @@ +# Pre-commit hooks to run tests and ensure code is cleaned. +# See https://pre-commit.com for more information +--- +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.6 + hooks: + - id: ruff + args: + - --fix + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: ["--keep-percent-format", "--py37-plus"] + - repo: https://github.com/psf/black + rev: 22.12.0 + hooks: + - id: black + args: ["--target-version", "py37", "--line-length", "100"] + - repo: https://github.com/PyCQA/isort + rev: 5.11.5 + hooks: + - id: isort + args: ["--profile", "black", "--line-length", "100"] + - repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + args: ["--max-line-length=100"] + - repo: https://github.com/codespell-project/codespell + rev: v2.2.2 + hooks: + - id: codespell + args: + - --ignore-words-list=fro,hass + - --skip="./.*,*.csv,*.json,*.ambr" + - --quiet-level=2 + exclude_types: [csv, json] + - repo: https://github.com/adrienverge/yamllint + rev: v1.32.0 + hooks: + - id: yamllint + args: ["-d {extends: relaxed, rules: {line-length: disable}}", "-s"] + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.7.1 + hooks: + - id: prettier + - repo: local + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + require_serial: true + - repo: https://github.com/Lucas-C/pre-commit-hooks-bandit + rev: v1.0.5 + hooks: + - id: python-bandit-vulnerability-check + name: bandit + args: [--skip, "B101", --recursive, mylib] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-executables-have-shebangs + stages: [manual] diff --git a/README.md b/README.md index 08ca188..107db01 100644 --- a/README.md +++ b/README.md @@ -3,19 +3,20 @@ This docker image could be used as an Woodpecker CI plugin to publish one (or more) Debian package on a Aptly repository using its API. It also could be used with Gitlab CI to define a publishing job. This plugin will try to : + - List all changes files in the specified directory and filter on the specified source package name (if specified) - Iter on detected changes files and foreach of then: - - the changes file is parsed to detect the source package name, the distribution and included files - - the repository name is computed (if not specified). __Format:__ `{prefix}_{distribution}_{component}`. __Note:__ if the default prefix is specified (`.`), it will not be used to compute the repository name. - - the current published distribution is retreived using APTLY Publish API to: - - check it was already manally published a first time - - check it used a snapshot kind of sources - - retreive other components source snapshot - - Upload the changes file and all its included files using APTLY File Upload API in a directory named as the source package - - Include the changes file using APTLY Local Repos API - - Compute a snapshot name for the repository based on the current date and the repository name. __Format:__ `YYYYMMDD-HHMMSS_{repository name}` - - Create a snapshot of the repository using APTLY Local Repos API - - Update the published distribution with this new snapshot as source of the specified component and keeping other components source snapshot. +- the changes file is parsed to detect the source package name, the distribution and included files +- the repository name is computed (if not specified). **Format:** `{prefix}_{distribution}_{component}`. **Note:** if the default prefix is specified (`.`), it will not be used to compute the repository name. +- the current published distribution is retrieved using APTLY Publish API to: + - check it was already manally published a first time + - check it used a snapshot kind of sources + - retrieve other components source snapshot +- Upload the changes file and all its included files using APTLY File Upload API in a directory named as the source package +- Include the changes file using APTLY Local Repos API +- Compute a snapshot name for the repository based on the current date and the repository name. **Format:** `YYYYMMDD-HHMMSS_{repository name}` +- Create a snapshot of the repository using APTLY Local Repos API +- Update the published distribution with this new snapshot as source of the specified component and keeping other components source snapshot. In case of error, it will exit with a detailed error message (within the limits of what is provided by the Aptly API). @@ -45,17 +46,18 @@ pipeline: force_overwrite: true ``` -__Parameters:__ -- __api_url:__ Your Aptly API URL (required) -- __api_username:__ Username to authenticate on your Aptly API (required) -- __api_password:__ Password to authenticate on your Aptly API (required) -- __prefix:__ The publishing prefix (optional, default: `.`) -- __repo_component:__ The component name to publish on (optional, default: `main`) -- __repo_name:__ The repository name to publish on. If not specified, it will be computed using the specified prefix and component and the detected package distribution. See above for details. -- __path:__ Path to the directory where files to publish are stored (optional, default: `dist`) -- __source_name:__ Name of the source package to publish (optional, default: all `changes` files are will be publish) -- __max_retries:__ The number of retry in case of error calling the Aptly API (optional, default: no retry) -- __force_overwrite:__ When publishing, overwrite files in `pool/` directory without notice (optional, default: false) +**Parameters:** + +- **api_url:** Your Aptly API URL (required) +- **api_username:** Username to authenticate on your Aptly API (required) +- **api_password:** Password to authenticate on your Aptly API (required) +- **prefix:** The publishing prefix (optional, default: `.`) +- **repo_component:** The component name to publish on (optional, default: `main`) +- **repo_name:** The repository name to publish on. If not specified, it will be computed using the specified prefix and component and the detected package distribution. See above for details. +- **path:** Path to the directory where files to publish are stored (optional, default: `dist`) +- **source_name:** Name of the source package to publish (optional, default: all `changes` files are will be publish) +- **max_retries:** The number of retry in case of error calling the Aptly API (optional, default: no retry) +- **force_overwrite:** When publishing, overwrite files in `pool/` directory without notice (optional, default: false) ## With Gitlab CI diff --git a/aptly-publish b/aptly-publish index 7ff7a9c..c9fb012 100755 --- a/aptly-publish +++ b/aptly-publish @@ -9,55 +9,51 @@ import os import re import sys +from debian_parser import PackagesParser from requests import Session from requests.adapters import HTTPAdapter from urllib3.util import Retry -from debian_parser import PackagesParser - def from_env(name, default=None): - """ Retrieve a parameter from environment """ - for var in (f'PLUGIN_{name}', f'APTLY_{name}'): + """Retrieve a parameter from environment""" + for var in (f"PLUGIN_{name}", f"APTLY_{name}"): if var in os.environ: return os.environ[var] return default # Handle parameters from environment -API_URL = from_env('API_URL', None) +API_URL = from_env("API_URL", None) if not API_URL: - print('API URL not provided') + print("API URL not provided") sys.exit(1) -API_USERNAME = from_env('API_USERNAME', None) +API_USERNAME = from_env("API_USERNAME", None) if not API_USERNAME: - print('API username not provided') + print("API username not provided") sys.exit(1) -API_PASSWORD = from_env('API_PASSWORD', None) +API_PASSWORD = from_env("API_PASSWORD", None) if not API_PASSWORD: - print('API password not provided') + print("API password not provided") sys.exit(1) -MAX_RETRY = from_env('MAX_RETRIES', None) +MAX_RETRY = from_env("MAX_RETRIES", None) -REPO_NAME = from_env('REPO_NAME', None) -PREFIX = from_env('PREFIX', '.') -REPO_COMPONENT = from_env('REPO_COMPONENT', 'main') -INPUT_PATH = from_env('PATH', 'dist') -SOURCE_NAME = from_env('SOURCE_PACKAGE_NAME', None) -FORCE_OVERWRITE = ( - from_env('FORCE_OVERWRITE', "false").lower() - in ["1", "true", "yes"] -) +REPO_NAME = from_env("REPO_NAME", None) +PREFIX = from_env("PREFIX", ".") +REPO_COMPONENT = from_env("REPO_COMPONENT", "main") +INPUT_PATH = from_env("PATH", "dist") +SOURCE_NAME = from_env("SOURCE_PACKAGE_NAME", None) +FORCE_OVERWRITE = from_env("FORCE_OVERWRITE", "false").lower() in ["1", "true", "yes"] # List changes files changes_files_regex = ( # pylint: disable=consider-using-f-string - re.compile(r'^%s_.*\.changes$' % SOURCE_NAME) - if SOURCE_NAME else - re.compile(r'^.*\.changes$') + re.compile(r"^%s_.*\.changes$" % SOURCE_NAME) + if SOURCE_NAME + else re.compile(r"^.*\.changes$") ) changes_files = [] try: @@ -75,7 +71,7 @@ except NotADirectoryError: sys.exit(1) if not changes_files: - print(f'No changes file found in {INPUT_PATH}') + print(f"No changes file found in {INPUT_PATH}") sys.exit(1) @@ -83,27 +79,24 @@ if not changes_files: session = Session() session.auth = (API_USERNAME, API_PASSWORD) if MAX_RETRY: - retries = Retry( - total=int(MAX_RETRY), - status_forcelist=list(range(500, 600)) - ) + retries = Retry(total=int(MAX_RETRY), status_forcelist=list(range(500, 600))) session.mount(API_URL, HTTPAdapter(max_retries=retries)) def get_repo_name(dist): - """ Compute and retreive repository name """ + """Compute and retrieve repository name""" if REPO_NAME: return REPO_NAME - value = f'{dist}_{REPO_COMPONENT}' + value = f"{dist}_{REPO_COMPONENT}" if PREFIX != ".": - value = f'{PREFIX}_{value}' + value = f"{PREFIX}_{value}" return value def parse_changes_file(filepath): - """ Parse changes file to detect distribution and included files """ + """Parse changes file to detect distribution and included files""" dirpath = os.path.dirname(filepath) - with open(filepath, "r", encoding="utf-8") as file_desc: + with open(filepath, encoding="utf-8") as file_desc: changes_file = file_desc.read() parser = PackagesParser(changes_file) @@ -112,198 +105,176 @@ def parse_changes_file(filepath): files = [] for infos in parser.parse(): for info in infos: - if info['tag'].lower() == 'files': - for line in info['value'].split(' '): + if info["tag"].lower() == "files": + for line in info["value"].split(" "): if not line: continue files.append(os.path.join(dirpath, line.split()[-1])) - if info['tag'].lower() == 'distribution': + if info["tag"].lower() == "distribution": if distribution: print( - 'More than one distribution found in changes file' - f'{os.path.basename(filepath)}.') + "More than one distribution found in changes file" + f"{os.path.basename(filepath)}." + ) sys.exit(1) - distribution = info['value'] - if info['tag'].lower() == 'source': + distribution = info["value"] + if info["tag"].lower() == "source": if package_name: print( - 'More than one source package name found in changes ' - f'file {os.path.basename(filepath)}.') + "More than one source package name found in changes " + f"file {os.path.basename(filepath)}." + ) sys.exit(1) - package_name = info['value'] + package_name = info["value"] if not package_name: print( - 'Fail to detect source package name from changes file ' - f'{os.path.basename(filepath)}.') + "Fail to detect source package name from changes file " f"{os.path.basename(filepath)}." + ) sys.exit(1) if not distribution: - print( - 'Fail to detect distribution from changes file ' - f'{os.path.basename(filepath)}.') + print("Fail to detect distribution from changes file " f"{os.path.basename(filepath)}.") sys.exit(1) if not files: - print( - 'No included file found in changes file' - f'{os.path.basename(filepath)}.') + print("No included file found in changes file" f"{os.path.basename(filepath)}.") sys.exit(1) return (package_name, distribution, files) def get_published_distribution_other_components_sources(distribution): - """ Retreive current published distribution using Aptly API """ - url = f'{API_URL}/publish' + """Retrieve current published distribution using Aptly API""" + url = f"{API_URL}/publish" result = session.get(url) if result.status_code != 200: print( - 'Fail to retreive current published distribution ' - f'{distribution} using Aptly API (HTTP code: {result.status_code})' + "Fail to retrieve current published distribution " + f"{distribution} using Aptly API (HTTP code: {result.status_code})" ) sys.exit(1) for data in result.json(): - if data['Prefix'] != PREFIX: + if data["Prefix"] != PREFIX: continue - if data['Distribution'] != distribution: + if data["Distribution"] != distribution: continue - if data['SourceKind'] != 'snapshot': + if data["SourceKind"] != "snapshot": print( - f'The distribution {distribution} currently published on ' + f"The distribution {distribution} currently published on " f'prefix "{PREFIX}" do not sourcing packages from snapshot(s) ' f'but from {data["SourceKind"]}.' ) sys.exit(1) - return [ - source for source in data['Sources'] - if source['Component'] != REPO_COMPONENT - ] + return [source for source in data["Sources"] if source["Component"] != REPO_COMPONENT] print( - f'Distribution {distribution} seem not currently published on prefix ' + f"Distribution {distribution} seem not currently published on prefix " f'"{PREFIX}". Please manually publish it a first time before using ' - f'{os.path.basename(sys.argv[0])}.' + f"{os.path.basename(sys.argv[0])}." ) sys.exit(1) - return False def upload_file(package_name, filepath): - """ Upload a file using Aptly API """ - url = f'{API_URL}/files/{package_name}' - with open(filepath, 'rb') as file_desc: - result = session.post(url, files={'file': file_desc}) + """Upload a file using Aptly API""" + url = f"{API_URL}/files/{package_name}" + with open(filepath, "rb") as file_desc: + result = session.post(url, files={"file": file_desc}) return ( - result.status_code == 200 and - f'{package_name}/{os.path.basename(filepath)}' in result.json() + result.status_code == 200 + and f"{package_name}/{os.path.basename(filepath)}" in result.json() ) def include_file(repo_name, package_name, changes_file): - """ Include a changes file using Aptly API """ - url = ( - f'{API_URL}/repos/{repo_name}/include/{package_name}/' - f'{os.path.basename(changes_file)}' - ) + """Include a changes file using Aptly API""" + url = f"{API_URL}/repos/{repo_name}/include/{package_name}/" f"{os.path.basename(changes_file)}" result = session.post(url) data = result.json() - if data.get('FailedFiles'): + if data.get("FailedFiles"): print() - print(f'Some error occurred including {changes_file}:') - print('Failed files:') - for failed_file in data['FailedFiles']: - print(f' - {os.path.basename(failed_file)}') - if data.get('Report', {}).get('Warnings'): - print('Warnings:') - print(' - %s' % '\n - '.join(data['Report']['Warnings'])) + print(f"Some error occurred including {changes_file}:") + print("Failed files:") + for failed_file in data["FailedFiles"]: + print(f" - {os.path.basename(failed_file)}") + if data.get("Report", {}).get("Warnings"): + print("Warnings:") + print(" - %s" % "\n - ".join(data["Report"]["Warnings"])) print() return False - if not ( - result.status_code == 200 and - data.get('Report', {}).get('Added') - ): - print( - f'Unknown error occurred including {changes_file}' - 'See APTLY API logs for details.') + if not (result.status_code == 200 and data.get("Report", {}).get("Added")): + print(f"Unknown error occurred including {changes_file}" "See APTLY API logs for details.") return False return True for changes_file in changes_files: - print(f'Handle changes file {changes_file}:') + print(f"Handle changes file {changes_file}:") package_name, distribution, filepaths = parse_changes_file(changes_file) filepaths += [changes_file] repo_name = get_repo_name(distribution) - other_components_sources = \ - get_published_distribution_other_components_sources(distribution) + other_components_sources = get_published_distribution_other_components_sources(distribution) - print(' - Upload files:') + print(" - Upload files:") for filepath in filepaths: if not upload_file(package_name, filepath): - print( - f' - {filepath}: fail to upload file. See APTLY API logs ' - 'for details.') + print(f" - {filepath}: fail to upload file. See APTLY API logs " "for details.") sys.exit(1) else: - print(f' - {filepath}') + print(f" - {filepath}") - print(f' - Include changes file {changes_file}:') + print(f" - Include changes file {changes_file}:") if include_file(repo_name, package_name, changes_file): - print(' - Changes file included') + print(" - Changes file included") else: sys.exit(1) # Create a snapshot of the repository - snap_name = datetime.datetime.now().strftime(f'%Y%m%d-%H%M%S_{repo_name}') + snap_name = datetime.datetime.now().strftime(f"%Y%m%d-%H%M%S_{repo_name}") print(f'Create new snapshot "{snap_name}" of repository "{repo_name}"') - url = f'{API_URL}/repos/{repo_name}/snapshots' - payload = {'Name': snap_name} + url = f"{API_URL}/repos/{repo_name}/snapshots" + payload = {"Name": snap_name} result = session.post(url, json=payload) try: data = result.json() except Exception: # pylint: disable=broad-except data = {} error = ( - result.status_code < 200 or - result.status_code > 299 or - data.get('Name') != snap_name or - not data.get('CreatedAt') + result.status_code < 200 + or result.status_code > 299 + or data.get("Name") != snap_name + or not data.get("CreatedAt") ) if error: print( f'Fail to create snapshot "{snap_name}" of repository ' - f'"{repo_name}". See APTLY API logs for details.') + f'"{repo_name}". See APTLY API logs for details.' + ) sys.exit(1) # Update published snapshot of the distribution print( f'Update published snapshot of distribution "{distribution}" to ' - f'"{snap_name}" (prefix: {PREFIX}, component: {REPO_COMPONENT})') + f'"{snap_name}" (prefix: {PREFIX}, component: {REPO_COMPONENT})' + ) if other_components_sources: - print('Note: keep other currently published components:') + print("Note: keep other currently published components:") for source in other_components_sources: print(f'- {source["Component"]}: {source["Name"]}') - url = f'{API_URL}/publish/:{PREFIX}/{distribution}' + url = f"{API_URL}/publish/:{PREFIX}/{distribution}" payload = { - 'Snapshots': other_components_sources + [ - { - 'Component': REPO_COMPONENT, - 'Name': snap_name - } - ], - 'ForceOverwrite': FORCE_OVERWRITE, + "Snapshots": other_components_sources + [{"Component": REPO_COMPONENT, "Name": snap_name}], + "ForceOverwrite": FORCE_OVERWRITE, } result = session.put(url, json=payload) - if ( - result.status_code < 200 or - result.status_code > 299 - ): + if result.status_code < 200 or result.status_code > 299: print( - 'Fail to update published snapshot of distribution ' + "Fail to update published snapshot of distribution " f'"{distribution}" to "{snap_name}" (prefix: {PREFIX}, ' - f'component: {REPO_COMPONENT}). See APTLY API logs for details.') + f"component: {REPO_COMPONENT}). See APTLY API logs for details." + ) sys.exit(1) print("Done.") diff --git a/docs.md b/docs.md index 66c99eb..5e20a42 100644 --- a/docs.md +++ b/docs.md @@ -14,36 +14,36 @@ Woodpecker CI plugin to publish one (or more) Debian package on a Aptly reposito ## Features This plugin will try to : + - List all changes files in the specified directory and filter on the specified source package name (if specified) - Iter on detected changes files and foreach of then: - - the changes file is parsed to detect the source package name, the distribution and included files - - the repository name is computed (if not specified). __Format:__ `{prefix}_{distribution}_{component}`. __Note:__ if the default prefix is specified (`.`), it will not be used to compute the repository name. - - the current published distribution is retreived using APTLY Publish API to: - - check it was already manally published a first time - - check it used a snapshot kind of sources - - retreive other components source snapshot - - Upload the changes file and all its included files using APTLY File Upload API in a directory named as the source package - - Include the changes file using APTLY Local Repos API - - Compute a snapshot name for the repository based on the current date and the repository name. __Format:__ `YYYYMMDD-HHMMSS_{repository name}` - - Create a snapshot of the repository using APTLY Local Repos API - - Update the published distribution with this new snapshot as source of the specified component and keeping other components source snapshot. +- the changes file is parsed to detect the source package name, the distribution and included files +- the repository name is computed (if not specified). **Format:** `{prefix}_{distribution}_{component}`. **Note:** if the default prefix is specified (`.`), it will not be used to compute the repository name. +- the current published distribution is retrieved using APTLY Publish API to: + - check it was already manally published a first time + - check it used a snapshot kind of sources + - retrieve other components source snapshot +- Upload the changes file and all its included files using APTLY File Upload API in a directory named as the source package +- Include the changes file using APTLY Local Repos API +- Compute a snapshot name for the repository based on the current date and the repository name. **Format:** `YYYYMMDD-HHMMSS_{repository name}` +- Create a snapshot of the repository using APTLY Local Repos API +- Update the published distribution with this new snapshot as source of the specified component and keeping other components source snapshot. In case of error, it will exit with a detailed error message (within the limits of what is provided by the Aptly API). ## Settings -| Settings Name | Default | Description -| --------------------------| ----------------- | -------------------------------------------- -| `api_url` | *none* | Your Aptly API URL (required) -| `api_username` | *none* | Username to authenticate on your Aptly API (required) -| `api_password` | *none* | Password to authenticate on your Aptly API (required) -| `prefix` | `.` | The publishing prefix -| `repo_component` | `main` | The component name to publish on -| `repo_name` | `{prefix}_{distribution}_{component}` | The repository name to publish on. If not specified, it will be computed using the specified prefix and component and the detected package distribution. See above for details. -| `path` | `dist` | Path to the directory where files to publish are stored -| `source_name` | *none* | Name of the source package to publish (optional, default: all `changes` files are will be publish) -| `max_retries` | *none* | The number of retry in case of error calling the Aptly API (optional, default: no retry) - +| Settings Name | Default | Description | +| ---------------- | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `api_url` | _none_ | Your Aptly API URL (required) | +| `api_username` | _none_ | Username to authenticate on your Aptly API (required) | +| `api_password` | _none_ | Password to authenticate on your Aptly API (required) | +| `prefix` | `.` | The publishing prefix | +| `repo_component` | `main` | The component name to publish on | +| `repo_name` | `{prefix}_{distribution}_{component}` | The repository name to publish on. If not specified, it will be computed using the specified prefix and component and the detected package distribution. See above for details. | +| `path` | `dist` | Path to the directory where files to publish are stored | +| `source_name` | _none_ | Name of the source package to publish (optional, default: all `changes` files are will be publish) | +| `max_retries` | _none_ | The number of retry in case of error calling the Aptly API (optional, default: no retry) | ## Example