From 82c459f910c3036b22c9f41472b622ef4bfdab1d Mon Sep 17 00:00:00 2001 From: Zbigniew Jędrzejewski-Szmek Date: Wed, 3 Jul 2024 16:01:03 +0200 Subject: tools/update-distro-hash: rename, fetch the repository if appropriate Let's rename the tool to tools/fetch-distro. It's useful to be able to fetch the distro directly. But when that functionality is added, the old name is confusing. Now --update/-u must be specified to update the commits. --reference-if-able is used to speed up the clone of debian. It saves about 75% of the download. --- tools/fetch-distro.py | 121 ++++++++++++++++++++++++++++++++++++++++++++ tools/update-distro-hash.py | 90 -------------------------------- 2 files changed, 121 insertions(+), 90 deletions(-) create mode 100755 tools/fetch-distro.py delete mode 100755 tools/update-distro-hash.py (limited to 'tools') diff --git a/tools/fetch-distro.py b/tools/fetch-distro.py new file mode 100755 index 0000000000..dacaf28bca --- /dev/null +++ b/tools/fetch-distro.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: LGPL-2.1-or-later + +""" +Check out pkg/{distribution}. +With -u, fetch commits, and if changed, commit the latest hash. +""" + +import argparse +import json +import shlex +import subprocess +from pathlib import Path + +def parse_args(): + p = argparse.ArgumentParser( + description=__doc__, + ) + p.add_argument( + 'distribution', + nargs='+', + ) + p.add_argument( + '--no-fetch', + dest='fetch', + action='store_false', + default=True, + ) + p.add_argument( + '--update', '-u', + action='store_true', + default=False, + ) + return p.parse_args() + +def read_config(distro: str): + cmd = ['mkosi', '--json', '-d', distro, 'summary'] + print(f"+ {shlex.join(cmd)}") + text = subprocess.check_output(cmd, text=True) + + data = json.loads(text) + images = {image["Image"]: image for image in data["Images"]} + return images["build"] + +def commit_file(distro: str, file: Path, commit: str, changes: str): + message = '\n'.join(( + f'mkosi: update {distro} commit reference', + '', + changes)) + + cmd = ['git', 'commit', '-m', message, str(file)] + print(f"+ {shlex.join(cmd)}") + subprocess.check_call(cmd) + +def checkout_distro(args, distro: str, config: dict): + dest = Path(f'pkg/{distro}') + if dest.exists(): + print(f'{dest} already exists.') + return + + url = config['Environment']['GIT_URL'] + branch = config['Environment']['GIT_BRANCH'] + + # Only debian uses source-git for now… + reference = [f'--reference-if-able=.'] if distro == 'debian' else [] + + cmd = [ + 'git', 'clone', url, + f'--branch={branch}', + dest.as_posix(), + *reference, + ] + print(f"+ {shlex.join(cmd)}") + subprocess.check_call(cmd) + + args.fetch = False # no need to fetch if we just cloned + +def update_distro(args, distro: str, config: dict): + cmd = ['git', '-C', f'pkg/{distro}', 'fetch'] + print(f"+ {shlex.join(cmd)}") + subprocess.check_call(cmd) + + branch = config['Environment']['GIT_BRANCH'] + old_commit = config['Environment']['GIT_COMMIT'] + + cmd = ['git', '-C', f'pkg/{distro}', 'rev-parse', f'refs/remotes/origin/{branch}'] + print(f"+ {shlex.join(cmd)}") + new_commit = subprocess.check_output(cmd, text=True).strip() + + if old_commit == new_commit: + print(f'{distro}: commit {new_commit!s} is still fresh') + return + + cmd = ['git', '-C', f'pkg/{distro}', 'log', '--graph', + '--pretty=oneline', '--no-decorate', '--abbrev-commit', '--abbrev=10', + f'{old_commit}..{new_commit}'] + print(f"+ {shlex.join(cmd)}") + changes = subprocess.check_output(cmd, text=True).strip() + + conf_dir = Path('mkosi.images/build/mkosi.conf.d') + files = conf_dir.glob('*/*.conf') + for file in files: + s = file.read_text() + if old_commit in s: + print(f'{distro}: {file}: found old hash, updating…') + new = s.replace(old_commit, new_commit) + assert new != s + file.write_text(new) + commit_file(distro, file, new_commit, changes) + break + else: + raise ValueError(f'{distro}: hash {new_commit} not found under {conf_dir}') + +if __name__ == '__main__': + args = parse_args() + + for distro in args.distribution: + config = read_config(distro) + checkout_distro(args, distro, config) + if args.update: + update_distro(args, distro, config) diff --git a/tools/update-distro-hash.py b/tools/update-distro-hash.py deleted file mode 100755 index 6f2d37f72e..0000000000 --- a/tools/update-distro-hash.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: LGPL-2.1-or-later - -""" -Fetch commits for pkg/{distribution} and, if changed, commit the latest hash. -""" - -import argparse -import json -import shlex -import subprocess -from pathlib import Path - -def parse_args(): - p = argparse.ArgumentParser( - description=__doc__, - ) - p.add_argument( - 'distribution', - nargs='+', - ) - p.add_argument( - '--no-fetch', - dest='fetch', - action='store_false', - default=True, - ) - return p.parse_args() - -def read_config(distro: str): - cmd = ['mkosi', '--json', '-d', distro, 'summary'] - print(f"+ {shlex.join(cmd)}") - text = subprocess.check_output(cmd, text=True) - - data = json.loads(text) - images = {image["Image"]: image for image in data["Images"]} - return images["build"] - -def commit_file(distro: str, file: Path, commit: str, changes: str): - message = '\n'.join(( - f'mkosi: update {distro} commit reference', - '', - changes)) - - cmd = ['git', 'commit', '-m', message, str(file)] - print(f"+ {shlex.join(cmd)}") - subprocess.check_call(cmd) - -def update_distro(args, distro: str): - cmd = ['git', '-C', f'pkg/{distro}', 'fetch'] - print(f"+ {shlex.join(cmd)}") - subprocess.check_call(cmd) - - config = read_config(distro) - - branch = config['Environment']['GIT_BRANCH'] - old_commit = config['Environment']['GIT_COMMIT'] - - cmd = ['git', '-C', f'pkg/{distro}', 'rev-parse', f'refs/remotes/origin/{branch}'] - print(f"+ {shlex.join(cmd)}") - new_commit = subprocess.check_output(cmd, text=True).strip() - - if old_commit == new_commit: - print(f'{distro}: commit {new_commit!s} is still fresh') - return - - cmd = ['git', '-C', f'pkg/{distro}', 'log', '--graph', - '--pretty=oneline', '--no-decorate', '--abbrev-commit', '--abbrev=10', - f'{old_commit}..{new_commit}'] - print(f"+ {shlex.join(cmd)}") - changes = subprocess.check_output(cmd, text=True).strip() - - conf_dir = Path('mkosi.images/build/mkosi.conf.d') - files = conf_dir.glob('*/*.conf') - for file in files: - s = file.read_text() - if old_commit in s: - print(f'{distro}: {file}: found old hash, updating…') - new = s.replace(old_commit, new_commit) - assert new != s - file.write_text(new) - commit_file(distro, file, new_commit, changes) - break - else: - raise ValueError(f'{distro}: hash {new_commit} not found under {conf_dir}') - -if __name__ == '__main__': - args = parse_args() - for distro in args.distribution: - update_distro(args, distro) -- cgit v1.2.3