Improvments to correctly handle prefix, component name & distribution to compute repository and snapshot name
The repository name could now be computed from prefix, distribution and component names. Format: {prefix}-{distribution}-{component} Note: if the default prefix is specified ("."), it will not be used to compute the repository name. So, all works is now done by itering on detected changes files. Foreach of then: - the changes file is parsed to detect the source package name, the distribution and included files - the repository name is computed (if not specified in environment) - the current published distribution is retreived using APTLY publish API to: - check it was already manally published a first time - check it used a snapshot kind of sources - retreive other components source snapshot - Upload the changes file and all its included files using APTLY File Upload API in a directory named as the source package - Include the changes file using APTLY Local Repos API - Compute a snapshot name for the repository based on the current date and the repository name. Format: YYYYMMDD-HHMMSS-{repository name} - Create a snapshot of the repository using APTLY Local Repos API - Update the published distribution with this new snapshot as source of the specified component (default: main) and keeping other components source snapshot.
This commit is contained in:
parent
6d7c03fb3d
commit
ce4d6a65e3
1 changed files with 122 additions and 45 deletions
135
aptly-publish
Normal file → Executable file
135
aptly-publish
Normal file → Executable file
|
@ -42,7 +42,7 @@ if not API_PASSWORD:
|
||||||
|
|
||||||
MAX_RETRY = from_env('MAX_RETRIES', None)
|
MAX_RETRY = from_env('MAX_RETRIES', None)
|
||||||
|
|
||||||
REPO_NAME = from_env('REPO_NAME', 'stable')
|
REPO_NAME = from_env('REPO_NAME', None)
|
||||||
PREFIX = from_env('PREFIX', '.')
|
PREFIX = from_env('PREFIX', '.')
|
||||||
REPO_COMPONENT = from_env('REPO_COMPONENT', 'main')
|
REPO_COMPONENT = from_env('REPO_COMPONENT', 'main')
|
||||||
INPUT_PATH = from_env('PATH', 'dist')
|
INPUT_PATH = from_env('PATH', 'dist')
|
||||||
|
@ -87,14 +87,25 @@ if MAX_RETRY:
|
||||||
session.mount(API_URL, HTTPAdapter(max_retries=retries))
|
session.mount(API_URL, HTTPAdapter(max_retries=retries))
|
||||||
|
|
||||||
|
|
||||||
# List and upload files from changes files
|
def get_repo_name(dist):
|
||||||
def list_files_in_changes_file(filepath):
|
""" Compute and retreive repository name """
|
||||||
""" List files included by a changes file """
|
if REPO_NAME:
|
||||||
|
return REPO_NAME
|
||||||
|
value = f'{dist}-{REPO_COMPONENT}'
|
||||||
|
if PREFIX != ".":
|
||||||
|
value = f'{PREFIX}-{value}'
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def parse_changes_file(filepath):
|
||||||
|
""" Parse changes file to detect distribution and included files """
|
||||||
dirpath = os.path.dirname(filepath)
|
dirpath = os.path.dirname(filepath)
|
||||||
with open(filepath, "r", encoding="utf-8") as file_desc:
|
with open(filepath, "r", encoding="utf-8") as file_desc:
|
||||||
changes_file = file_desc.read()
|
changes_file = file_desc.read()
|
||||||
|
|
||||||
parser = PackagesParser(changes_file)
|
parser = PackagesParser(changes_file)
|
||||||
|
package_name = None
|
||||||
|
distribution = None
|
||||||
files = []
|
files = []
|
||||||
for infos in parser.parse():
|
for infos in parser.parse():
|
||||||
for info in infos:
|
for info in infos:
|
||||||
|
@ -104,14 +115,74 @@ def list_files_in_changes_file(filepath):
|
||||||
continue
|
continue
|
||||||
files.append(os.path.join(dirpath, line.split()[-1]))
|
files.append(os.path.join(dirpath, line.split()[-1]))
|
||||||
if info['tag'].lower() == 'distribution':
|
if info['tag'].lower() == 'distribution':
|
||||||
if info['value'] not in DISTRIBUTIONS:
|
if distribution:
|
||||||
DISTRIBUTIONS.append(info['value'])
|
print(
|
||||||
return files
|
'More than one distribution found in changes file'
|
||||||
|
f'{os.path.basename(filepath)}.')
|
||||||
|
sys.exit(1)
|
||||||
|
distribution = info['value']
|
||||||
|
if info['tag'].lower() == 'source':
|
||||||
|
if package_name:
|
||||||
|
print(
|
||||||
|
'More than one source package name found in changes '
|
||||||
|
f'file {os.path.basename(filepath)}.')
|
||||||
|
sys.exit(1)
|
||||||
|
package_name = info['value']
|
||||||
|
|
||||||
|
if not package_name:
|
||||||
|
print(
|
||||||
|
'Fail to detect source package name from changes file '
|
||||||
|
f'{os.path.basename(filepath)}.')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if not distribution:
|
||||||
|
print(
|
||||||
|
'Fail to detect distribution from changes file '
|
||||||
|
f'{os.path.basename(filepath)}.')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if not files:
|
||||||
|
print(
|
||||||
|
'No included file found in changes file'
|
||||||
|
f'{os.path.basename(filepath)}.')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
return (package_name, distribution, files)
|
||||||
|
|
||||||
|
|
||||||
def changes_file2package_name(changes_file):
|
def get_published_distribution_other_components_sources(distribution):
|
||||||
""" Retrieve package name from changes file name """
|
""" Retreive current published distribution using Aptly API """
|
||||||
return os.path.basename(changes_file).split('_')[0]
|
url = f'{API_URL}/publish'
|
||||||
|
result = session.get(url)
|
||||||
|
if result.status_code != 200:
|
||||||
|
print(
|
||||||
|
'Fail to retreive current published distribution '
|
||||||
|
f'{distribution} using Aptly API (HTTP code: {result.status_code})'
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
for data in result.json():
|
||||||
|
if data['Prefix'] != PREFIX:
|
||||||
|
continue
|
||||||
|
if data['Distribution'] != distribution:
|
||||||
|
continue
|
||||||
|
if data['SourceKind'] != 'snapshot':
|
||||||
|
print(
|
||||||
|
f'The distribution {distribution} currently published on '
|
||||||
|
f'prefix "{PREFIX}" do not sourcing packages from snapshot(s) '
|
||||||
|
f'but from {data["SourceKind"]}.'
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
return [
|
||||||
|
source for source in data['Sources']
|
||||||
|
if source['Component'] != REPO_COMPONENT
|
||||||
|
]
|
||||||
|
print(
|
||||||
|
f'Distribution {distribution} seem not currently published on prefix '
|
||||||
|
f'"{PREFIX}". Please manually publish it a first time before using '
|
||||||
|
f'{os.path.basename(sys.argv[0])}.'
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def upload_file(package_name, filepath):
|
def upload_file(package_name, filepath):
|
||||||
|
@ -125,10 +196,10 @@ def upload_file(package_name, filepath):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def include_file(package_name, changes_file):
|
def include_file(repo_name, package_name, changes_file):
|
||||||
""" Include a changes file using Aptly API """
|
""" Include a changes file using Aptly API """
|
||||||
url = (
|
url = (
|
||||||
f'{API_URL}/repos/{REPO_NAME}/include/{package_name}/'
|
f'{API_URL}/repos/{repo_name}/include/{package_name}/'
|
||||||
f'{os.path.basename(changes_file)}'
|
f'{os.path.basename(changes_file)}'
|
||||||
)
|
)
|
||||||
result = session.post(url)
|
result = session.post(url)
|
||||||
|
@ -156,9 +227,14 @@ def include_file(package_name, changes_file):
|
||||||
|
|
||||||
|
|
||||||
for changes_file in changes_files:
|
for changes_file in changes_files:
|
||||||
package_name = changes_file2package_name(changes_file)
|
|
||||||
print(f'Handle changes file {changes_file}:')
|
print(f'Handle changes file {changes_file}:')
|
||||||
filepaths = [changes_file] + list_files_in_changes_file(changes_file)
|
package_name, distribution, filepaths = parse_changes_file(changes_file)
|
||||||
|
filepaths += [changes_file]
|
||||||
|
repo_name = get_repo_name(distribution)
|
||||||
|
|
||||||
|
other_components_sources = \
|
||||||
|
get_published_distribution_other_components_sources(distribution)
|
||||||
|
|
||||||
print(' - Upload files:')
|
print(' - Upload files:')
|
||||||
for filepath in filepaths:
|
for filepath in filepaths:
|
||||||
if not upload_file(package_name, filepath):
|
if not upload_file(package_name, filepath):
|
||||||
|
@ -170,17 +246,16 @@ for changes_file in changes_files:
|
||||||
print(f' - {filepath}')
|
print(f' - {filepath}')
|
||||||
|
|
||||||
print(f' - Include changes file {changes_file}:')
|
print(f' - Include changes file {changes_file}:')
|
||||||
if include_file(package_name, changes_file):
|
if include_file(repo_name, package_name, changes_file):
|
||||||
print(' - Changes file included')
|
print(' - Changes file included')
|
||||||
else:
|
else:
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
# Create a snapshot of the repository
|
# Create a snapshot of the repository
|
||||||
snap_name = datetime.datetime.now().strftime(f'%Y%m%d-%H%M%S_{REPO_NAME}')
|
snap_name = datetime.datetime.now().strftime(f'%Y%m%d-%H%M%S-{repo_name}')
|
||||||
print(f'Create new snapshot "{snap_name}" of repository "{REPO_NAME}"')
|
print(f'Create new snapshot "{snap_name}" of repository "{repo_name}"')
|
||||||
|
|
||||||
url = f'{API_URL}/repos/{REPO_NAME}/snapshots'
|
url = f'{API_URL}/repos/{repo_name}/snapshots'
|
||||||
payload = {'Name': snap_name}
|
payload = {'Name': snap_name}
|
||||||
result = session.post(url, json=payload)
|
result = session.post(url, json=payload)
|
||||||
try:
|
try:
|
||||||
|
@ -195,19 +270,21 @@ error = (
|
||||||
)
|
)
|
||||||
if error:
|
if error:
|
||||||
print(
|
print(
|
||||||
f'Fail to create snapshot "{snap_name}" of repository "{REPO_NAME}".'
|
f'Fail to create snapshot "{snap_name}" of repository '
|
||||||
'See APTLY API logs for details.')
|
f'"{repo_name}". See APTLY API logs for details.')
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Update published snapshot of the distribution
|
||||||
# Update published snapshot of repository for each distributions
|
|
||||||
for distribution in DISTRIBUTIONS:
|
|
||||||
print(
|
print(
|
||||||
f'Update published snapshot of distribution "{distribution}" to '
|
f'Update published snapshot of distribution "{distribution}" to '
|
||||||
f'"{snap_name}" (prefix: {PREFIX})')
|
f'"{snap_name}" (prefix: {PREFIX}, component: {REPO_COMPONENT})')
|
||||||
|
if other_components_sources:
|
||||||
|
print('Note: keep other currently published components:')
|
||||||
|
for source in other_components_sources:
|
||||||
|
print(f'- {source["Component"]}: {source["Name"]}')
|
||||||
url = f'{API_URL}/publish/:{PREFIX}/{distribution}'
|
url = f'{API_URL}/publish/:{PREFIX}/{distribution}'
|
||||||
payload = {
|
payload = {
|
||||||
'Snapshots': [
|
'Snapshots': other_components_sources + [
|
||||||
{
|
{
|
||||||
'Component': REPO_COMPONENT,
|
'Component': REPO_COMPONENT,
|
||||||
'Name': snap_name
|
'Name': snap_name
|
||||||
|
@ -220,9 +297,9 @@ for distribution in DISTRIBUTIONS:
|
||||||
result.status_code > 299
|
result.status_code > 299
|
||||||
):
|
):
|
||||||
print(
|
print(
|
||||||
f'Fail to update published snapshot of distribution "{distribution}" '
|
'Fail to update published snapshot of distribution '
|
||||||
f'to "{snap_name}" (prefix: {PREFIX}). See APTLY API logs for '
|
f'"{distribution}" to "{snap_name}" (prefix: {PREFIX}, '
|
||||||
'details.')
|
f'component: {REPO_COMPONENT}). See APTLY API logs for details.')
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
print("Done.")
|
print("Done.")
|
||||||
|
|
Loading…
Reference in a new issue