From ce4d6a65e36e0951fbb58a267f2b3a43e0778fba Mon Sep 17 00:00:00 2001 From: Benjamin Renard Date: Tue, 13 Dec 2022 20:14:01 +0100 Subject: [PATCH] Improvments to correctly handle prefix, component name & distribution to compute repository and snapshot name The repository name could now be computed from prefix, distribution and component names. Format: {prefix}-{distribution}-{component} Note: if the default prefix is specified ("."), it will not be used to compute the repository name. So, all works is now done by itering on detected changes files. Foreach of then: - the changes file is parsed to detect the source package name, the distribution and included files - the repository name is computed (if not specified in environment) - the current published distribution is retreived using APTLY publish API to: - check it was already manally published a first time - check it used a snapshot kind of sources - retreive other components source snapshot - Upload the changes file and all its included files using APTLY File Upload API in a directory named as the source package - Include the changes file using APTLY Local Repos API - Compute a snapshot name for the repository based on the current date and the repository name. Format: YYYYMMDD-HHMMSS-{repository name} - Create a snapshot of the repository using APTLY Local Repos API - Update the published distribution with this new snapshot as source of the specified component (default: main) and keeping other components source snapshot. --- aptly-publish | 167 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 122 insertions(+), 45 deletions(-) mode change 100644 => 100755 aptly-publish diff --git a/aptly-publish b/aptly-publish old mode 100644 new mode 100755 index 667c080..33352d7 --- a/aptly-publish +++ b/aptly-publish @@ -42,7 +42,7 @@ if not API_PASSWORD: MAX_RETRY = from_env('MAX_RETRIES', None) -REPO_NAME = from_env('REPO_NAME', 'stable') +REPO_NAME = from_env('REPO_NAME', None) PREFIX = from_env('PREFIX', '.') REPO_COMPONENT = from_env('REPO_COMPONENT', 'main') INPUT_PATH = from_env('PATH', 'dist') @@ -87,14 +87,25 @@ if MAX_RETRY: session.mount(API_URL, HTTPAdapter(max_retries=retries)) -# List and upload files from changes files -def list_files_in_changes_file(filepath): - """ List files included by a changes file """ +def get_repo_name(dist): + """ Compute and retreive repository name """ + if REPO_NAME: + return REPO_NAME + value = f'{dist}-{REPO_COMPONENT}' + if PREFIX != ".": + value = f'{PREFIX}-{value}' + return value + + +def parse_changes_file(filepath): + """ Parse changes file to detect distribution and included files """ dirpath = os.path.dirname(filepath) with open(filepath, "r", encoding="utf-8") as file_desc: changes_file = file_desc.read() parser = PackagesParser(changes_file) + package_name = None + distribution = None files = [] for infos in parser.parse(): for info in infos: @@ -104,14 +115,74 @@ def list_files_in_changes_file(filepath): continue files.append(os.path.join(dirpath, line.split()[-1])) if info['tag'].lower() == 'distribution': - if info['value'] not in DISTRIBUTIONS: - DISTRIBUTIONS.append(info['value']) - return files + if distribution: + print( + 'More than one distribution found in changes file' + f'{os.path.basename(filepath)}.') + sys.exit(1) + distribution = info['value'] + if info['tag'].lower() == 'source': + if package_name: + print( + 'More than one source package name found in changes ' + f'file {os.path.basename(filepath)}.') + sys.exit(1) + package_name = info['value'] + + if not package_name: + print( + 'Fail to detect source package name from changes file ' + f'{os.path.basename(filepath)}.') + sys.exit(1) + + if not distribution: + print( + 'Fail to detect distribution from changes file ' + f'{os.path.basename(filepath)}.') + sys.exit(1) + + if not files: + print( + 'No included file found in changes file' + f'{os.path.basename(filepath)}.') + sys.exit(1) + + return (package_name, distribution, files) -def changes_file2package_name(changes_file): - """ Retrieve package name from changes file name """ - return os.path.basename(changes_file).split('_')[0] +def get_published_distribution_other_components_sources(distribution): + """ Retreive current published distribution using Aptly API """ + url = f'{API_URL}/publish' + result = session.get(url) + if result.status_code != 200: + print( + 'Fail to retreive current published distribution ' + f'{distribution} using Aptly API (HTTP code: {result.status_code})' + ) + sys.exit(1) + for data in result.json(): + if data['Prefix'] != PREFIX: + continue + if data['Distribution'] != distribution: + continue + if data['SourceKind'] != 'snapshot': + print( + f'The distribution {distribution} currently published on ' + f'prefix "{PREFIX}" do not sourcing packages from snapshot(s) ' + f'but from {data["SourceKind"]}.' + ) + sys.exit(1) + return [ + source for source in data['Sources'] + if source['Component'] != REPO_COMPONENT + ] + print( + f'Distribution {distribution} seem not currently published on prefix ' + f'"{PREFIX}". Please manually publish it a first time before using ' + f'{os.path.basename(sys.argv[0])}.' + ) + sys.exit(1) + return False def upload_file(package_name, filepath): @@ -125,10 +196,10 @@ def upload_file(package_name, filepath): ) -def include_file(package_name, changes_file): +def include_file(repo_name, package_name, changes_file): """ Include a changes file using Aptly API """ url = ( - f'{API_URL}/repos/{REPO_NAME}/include/{package_name}/' + f'{API_URL}/repos/{repo_name}/include/{package_name}/' f'{os.path.basename(changes_file)}' ) result = session.post(url) @@ -156,9 +227,14 @@ def include_file(package_name, changes_file): for changes_file in changes_files: - package_name = changes_file2package_name(changes_file) print(f'Handle changes file {changes_file}:') - filepaths = [changes_file] + list_files_in_changes_file(changes_file) + package_name, distribution, filepaths = parse_changes_file(changes_file) + filepaths += [changes_file] + repo_name = get_repo_name(distribution) + + other_components_sources = \ + get_published_distribution_other_components_sources(distribution) + print(' - Upload files:') for filepath in filepaths: if not upload_file(package_name, filepath): @@ -170,44 +246,45 @@ for changes_file in changes_files: print(f' - {filepath}') print(f' - Include changes file {changes_file}:') - if include_file(package_name, changes_file): + if include_file(repo_name, package_name, changes_file): print(' - Changes file included') else: sys.exit(1) + # Create a snapshot of the repository + snap_name = datetime.datetime.now().strftime(f'%Y%m%d-%H%M%S-{repo_name}') + print(f'Create new snapshot "{snap_name}" of repository "{repo_name}"') -# Create a snapshot of the repository -snap_name = datetime.datetime.now().strftime(f'%Y%m%d-%H%M%S_{REPO_NAME}') -print(f'Create new snapshot "{snap_name}" of repository "{REPO_NAME}"') + url = f'{API_URL}/repos/{repo_name}/snapshots' + payload = {'Name': snap_name} + result = session.post(url, json=payload) + try: + data = result.json() + except Exception: # pylint: disable=broad-except + data = {} + error = ( + result.status_code < 200 or + result.status_code > 299 or + data.get('Name') != snap_name or + not data.get('CreatedAt') + ) + if error: + print( + f'Fail to create snapshot "{snap_name}" of repository ' + f'"{repo_name}". See APTLY API logs for details.') + sys.exit(1) -url = f'{API_URL}/repos/{REPO_NAME}/snapshots' -payload = {'Name': snap_name} -result = session.post(url, json=payload) -try: - data = result.json() -except Exception: # pylint: disable=broad-except - data = {} -error = ( - result.status_code < 200 or - result.status_code > 299 or - data.get('Name') != snap_name or - not data.get('CreatedAt') -) -if error: - print( - f'Fail to create snapshot "{snap_name}" of repository "{REPO_NAME}".' - 'See APTLY API logs for details.') - sys.exit(1) - - -# Update published snapshot of repository for each distributions -for distribution in DISTRIBUTIONS: + # Update published snapshot of the distribution print( f'Update published snapshot of distribution "{distribution}" to ' - f'"{snap_name}" (prefix: {PREFIX})') + f'"{snap_name}" (prefix: {PREFIX}, component: {REPO_COMPONENT})') + if other_components_sources: + print('Note: keep other currently published components:') + for source in other_components_sources: + print(f'- {source["Component"]}: {source["Name"]}') url = f'{API_URL}/publish/:{PREFIX}/{distribution}' payload = { - 'Snapshots': [ + 'Snapshots': other_components_sources + [ { 'Component': REPO_COMPONENT, 'Name': snap_name @@ -220,9 +297,9 @@ for distribution in DISTRIBUTIONS: result.status_code > 299 ): print( - f'Fail to update published snapshot of distribution "{distribution}" ' - f'to "{snap_name}" (prefix: {PREFIX}). See APTLY API logs for ' - 'details.') + 'Fail to update published snapshot of distribution ' + f'"{distribution}" to "{snap_name}" (prefix: {PREFIX}, ' + f'component: {REPO_COMPONENT}). See APTLY API logs for details.') sys.exit(1) print("Done.")