Improvments to correctly handle prefix, component name & distribution to compute repository and snapshot name

The repository name could now be computed from prefix, distribution and
component names. Format:
  {prefix}-{distribution}-{component}

Note: if the default prefix is specified ("."), it will not be used to
compute the repository name.

So, all works is now done by itering on detected changes files. Foreach
of then:
 - the changes file is parsed to detect the source package name, the
   distribution and included files
 - the repository name is computed (if not specified in environment)
 - the current published distribution is retreived using APTLY publish
   API to:
   - check it was already manally published a first time
   - check it used a snapshot kind of sources
   - retreive other components source snapshot
 - Upload the changes file and all its included files using APTLY File
   Upload API in a directory named as the source package
 - Include the changes file using APTLY Local Repos API
 - Compute a snapshot name for the repository based on the current date
   and the repository name. Format: YYYYMMDD-HHMMSS-{repository name}
 - Create a snapshot of the repository using APTLY Local Repos API
 - Update the published distribution with this new snapshot as source of
   the specified component (default: main) and keeping other components
   source snapshot.
This commit is contained in:
Benjamin Renard 2022-12-13 20:14:01 +01:00
parent 6d7c03fb3d
commit ce4d6a65e3

167
aptly-publish Normal file → Executable file
View file

@ -42,7 +42,7 @@ if not API_PASSWORD:
MAX_RETRY = from_env('MAX_RETRIES', None) MAX_RETRY = from_env('MAX_RETRIES', None)
REPO_NAME = from_env('REPO_NAME', 'stable') REPO_NAME = from_env('REPO_NAME', None)
PREFIX = from_env('PREFIX', '.') PREFIX = from_env('PREFIX', '.')
REPO_COMPONENT = from_env('REPO_COMPONENT', 'main') REPO_COMPONENT = from_env('REPO_COMPONENT', 'main')
INPUT_PATH = from_env('PATH', 'dist') INPUT_PATH = from_env('PATH', 'dist')
@ -87,14 +87,25 @@ if MAX_RETRY:
session.mount(API_URL, HTTPAdapter(max_retries=retries)) session.mount(API_URL, HTTPAdapter(max_retries=retries))
# List and upload files from changes files def get_repo_name(dist):
def list_files_in_changes_file(filepath): """ Compute and retreive repository name """
""" List files included by a changes file """ if REPO_NAME:
return REPO_NAME
value = f'{dist}-{REPO_COMPONENT}'
if PREFIX != ".":
value = f'{PREFIX}-{value}'
return value
def parse_changes_file(filepath):
""" Parse changes file to detect distribution and included files """
dirpath = os.path.dirname(filepath) dirpath = os.path.dirname(filepath)
with open(filepath, "r", encoding="utf-8") as file_desc: with open(filepath, "r", encoding="utf-8") as file_desc:
changes_file = file_desc.read() changes_file = file_desc.read()
parser = PackagesParser(changes_file) parser = PackagesParser(changes_file)
package_name = None
distribution = None
files = [] files = []
for infos in parser.parse(): for infos in parser.parse():
for info in infos: for info in infos:
@ -104,14 +115,74 @@ def list_files_in_changes_file(filepath):
continue continue
files.append(os.path.join(dirpath, line.split()[-1])) files.append(os.path.join(dirpath, line.split()[-1]))
if info['tag'].lower() == 'distribution': if info['tag'].lower() == 'distribution':
if info['value'] not in DISTRIBUTIONS: if distribution:
DISTRIBUTIONS.append(info['value']) print(
return files 'More than one distribution found in changes file'
f'{os.path.basename(filepath)}.')
sys.exit(1)
distribution = info['value']
if info['tag'].lower() == 'source':
if package_name:
print(
'More than one source package name found in changes '
f'file {os.path.basename(filepath)}.')
sys.exit(1)
package_name = info['value']
if not package_name:
print(
'Fail to detect source package name from changes file '
f'{os.path.basename(filepath)}.')
sys.exit(1)
if not distribution:
print(
'Fail to detect distribution from changes file '
f'{os.path.basename(filepath)}.')
sys.exit(1)
if not files:
print(
'No included file found in changes file'
f'{os.path.basename(filepath)}.')
sys.exit(1)
return (package_name, distribution, files)
def changes_file2package_name(changes_file): def get_published_distribution_other_components_sources(distribution):
""" Retrieve package name from changes file name """ """ Retreive current published distribution using Aptly API """
return os.path.basename(changes_file).split('_')[0] url = f'{API_URL}/publish'
result = session.get(url)
if result.status_code != 200:
print(
'Fail to retreive current published distribution '
f'{distribution} using Aptly API (HTTP code: {result.status_code})'
)
sys.exit(1)
for data in result.json():
if data['Prefix'] != PREFIX:
continue
if data['Distribution'] != distribution:
continue
if data['SourceKind'] != 'snapshot':
print(
f'The distribution {distribution} currently published on '
f'prefix "{PREFIX}" do not sourcing packages from snapshot(s) '
f'but from {data["SourceKind"]}.'
)
sys.exit(1)
return [
source for source in data['Sources']
if source['Component'] != REPO_COMPONENT
]
print(
f'Distribution {distribution} seem not currently published on prefix '
f'"{PREFIX}". Please manually publish it a first time before using '
f'{os.path.basename(sys.argv[0])}.'
)
sys.exit(1)
return False
def upload_file(package_name, filepath): def upload_file(package_name, filepath):
@ -125,10 +196,10 @@ def upload_file(package_name, filepath):
) )
def include_file(package_name, changes_file): def include_file(repo_name, package_name, changes_file):
""" Include a changes file using Aptly API """ """ Include a changes file using Aptly API """
url = ( url = (
f'{API_URL}/repos/{REPO_NAME}/include/{package_name}/' f'{API_URL}/repos/{repo_name}/include/{package_name}/'
f'{os.path.basename(changes_file)}' f'{os.path.basename(changes_file)}'
) )
result = session.post(url) result = session.post(url)
@ -156,9 +227,14 @@ def include_file(package_name, changes_file):
for changes_file in changes_files: for changes_file in changes_files:
package_name = changes_file2package_name(changes_file)
print(f'Handle changes file {changes_file}:') print(f'Handle changes file {changes_file}:')
filepaths = [changes_file] + list_files_in_changes_file(changes_file) package_name, distribution, filepaths = parse_changes_file(changes_file)
filepaths += [changes_file]
repo_name = get_repo_name(distribution)
other_components_sources = \
get_published_distribution_other_components_sources(distribution)
print(' - Upload files:') print(' - Upload files:')
for filepath in filepaths: for filepath in filepaths:
if not upload_file(package_name, filepath): if not upload_file(package_name, filepath):
@ -170,44 +246,45 @@ for changes_file in changes_files:
print(f' - {filepath}') print(f' - {filepath}')
print(f' - Include changes file {changes_file}:') print(f' - Include changes file {changes_file}:')
if include_file(package_name, changes_file): if include_file(repo_name, package_name, changes_file):
print(' - Changes file included') print(' - Changes file included')
else: else:
sys.exit(1) sys.exit(1)
# Create a snapshot of the repository
snap_name = datetime.datetime.now().strftime(f'%Y%m%d-%H%M%S-{repo_name}')
print(f'Create new snapshot "{snap_name}" of repository "{repo_name}"')
# Create a snapshot of the repository url = f'{API_URL}/repos/{repo_name}/snapshots'
snap_name = datetime.datetime.now().strftime(f'%Y%m%d-%H%M%S_{REPO_NAME}') payload = {'Name': snap_name}
print(f'Create new snapshot "{snap_name}" of repository "{REPO_NAME}"') result = session.post(url, json=payload)
try:
data = result.json()
except Exception: # pylint: disable=broad-except
data = {}
error = (
result.status_code < 200 or
result.status_code > 299 or
data.get('Name') != snap_name or
not data.get('CreatedAt')
)
if error:
print(
f'Fail to create snapshot "{snap_name}" of repository '
f'"{repo_name}". See APTLY API logs for details.')
sys.exit(1)
url = f'{API_URL}/repos/{REPO_NAME}/snapshots' # Update published snapshot of the distribution
payload = {'Name': snap_name}
result = session.post(url, json=payload)
try:
data = result.json()
except Exception: # pylint: disable=broad-except
data = {}
error = (
result.status_code < 200 or
result.status_code > 299 or
data.get('Name') != snap_name or
not data.get('CreatedAt')
)
if error:
print(
f'Fail to create snapshot "{snap_name}" of repository "{REPO_NAME}".'
'See APTLY API logs for details.')
sys.exit(1)
# Update published snapshot of repository for each distributions
for distribution in DISTRIBUTIONS:
print( print(
f'Update published snapshot of distribution "{distribution}" to ' f'Update published snapshot of distribution "{distribution}" to '
f'"{snap_name}" (prefix: {PREFIX})') f'"{snap_name}" (prefix: {PREFIX}, component: {REPO_COMPONENT})')
if other_components_sources:
print('Note: keep other currently published components:')
for source in other_components_sources:
print(f'- {source["Component"]}: {source["Name"]}')
url = f'{API_URL}/publish/:{PREFIX}/{distribution}' url = f'{API_URL}/publish/:{PREFIX}/{distribution}'
payload = { payload = {
'Snapshots': [ 'Snapshots': other_components_sources + [
{ {
'Component': REPO_COMPONENT, 'Component': REPO_COMPONENT,
'Name': snap_name 'Name': snap_name
@ -220,9 +297,9 @@ for distribution in DISTRIBUTIONS:
result.status_code > 299 result.status_code > 299
): ):
print( print(
f'Fail to update published snapshot of distribution "{distribution}" ' 'Fail to update published snapshot of distribution '
f'to "{snap_name}" (prefix: {PREFIX}). See APTLY API logs for ' f'"{distribution}" to "{snap_name}" (prefix: {PREFIX}, '
'details.') f'component: {REPO_COMPONENT}). See APTLY API logs for details.')
sys.exit(1) sys.exit(1)
print("Done.") print("Done.")