From 742dadb285eefe90aab1995a2b3a657390fe221c Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 1 Nov 2023 18:14:34 -0400 Subject: cephadm: update the build.py script to work on multiple distros Unfortunately, a single simple call to pip does not work on all the distributions that ceph is built on. In particular, Ubuntu 20.04 and Ubuntu 22.04 come with pip versions that can not correctly handle disabling wheels and installing Jinja2 (it tries to use the markupsafe dependency before it is installed). This can be worked around by using a virtual env and updating pip before proceeding. However, this is not enough because CentOS/RHEL 8 uses python 3.6 and there is no version of pip that supports 3.6 that we can update to that is new enough to fix the issue with disabling wheels. The workaround in this case is to install each dependency one at a time through multiple calls to pip. Because of this extra complexity is it simpler to eschew the use of a requirements.txt file in build.py entirely. Thus the zipapp is built using build.py only. Requirements files for cephadm are for setting up the tox environments *only*. For completeness a new option is added that gives the caller control over when build.py uses a virtualenv or not. Thus the build.py script requires at least one of: a working pip that handles disabling wheels; or, a virtualenv (venv) and the ability to update to a working version of pip. If the list of distros ceph supports (and the python versions they use) ever becomes simpler/newer some of this complexity could be removed from the build.py script. Signed-off-by: John Mulligan --- src/cephadm/build.py | 141 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 120 insertions(+), 21 deletions(-) (limited to 'src/cephadm/build.py') diff --git a/src/cephadm/build.py b/src/cephadm/build.py index f5a2ce27479..87d990e72a3 100755 --- a/src/cephadm/build.py +++ b/src/cephadm/build.py @@ -7,12 +7,14 @@ import argparse import compileall +import enum import logging import os import pathlib import shutil import subprocess import tempfile +import shlex import sys HAS_ZIPAPP = False @@ -27,6 +29,20 @@ except ImportError: log = logging.getLogger(__name__) +PY36_REQUIREMENTS = [ + 'MarkupSafe >= 2.0.1, <2.2', + 'Jinja2 >= 3.0.2, <3.2', +] +PY_REQUIREMENTS = [ + 'MarkupSafe >= 2.1.3, <2.2', + 'Jinja2 >= 3.1.2, <3.2', +] +# IMPORTANT to be fully compatible with all the distros ceph is built for we +# need to work around various old versions of python/pip. As such it's easier +# to repeat our requirements in this script than it is to parse zipapp-reqs.txt. +# You *must* keep the PY_REQUIREMENTS list in sync with the contents of +# zipapp-reqs.txt manually. + _VALID_VERS_VARS = [ "CEPH_GIT_VER", "CEPH_GIT_NICE_VER", @@ -36,6 +52,35 @@ _VALID_VERS_VARS = [ ] +class PipEnv(enum.Enum): + never = enum.auto() + auto = enum.auto() + required = enum.auto() + + @property + def enabled(self): + return self == self.auto or self == self.required + + +class Config: + def __init__(self, cli_args): + self.cli_args = cli_args + self._maj_min = sys.version_info[0:2] + self.install_dependencies = True + if self._maj_min == (3, 6): + self.pip_split = True + self.requirements = PY36_REQUIREMENTS + else: + self.pip_split = False + self.requirements = PY_REQUIREMENTS + self.pip_venv = PipEnv[cli_args.pip_use_venv] + + +def _run(command, *args, **kwargs): + log.info('Running cmd: %s', ' '.join(shlex.quote(str(c)) for c in command)) + return subprocess.run(command, *args, **kwargs) + + def _reexec(python): """Switch to the selected version of python by exec'ing into the desired python path. @@ -54,14 +99,14 @@ def _did_rexec(): return bool(os.environ.get("_BUILD_PYTHON_SET", "")) -def _build(dest, src, versioning_vars=None): +def _build(dest, src, config): """Build the binary.""" os.chdir(src) tempdir = pathlib.Path(tempfile.mkdtemp(suffix=".cephadm.build")) log.debug("working in %s", tempdir) try: - if os.path.isfile("requirements.txt"): - _install_deps(tempdir) + if config.install_dependencies: + _install_deps(tempdir, config) log.info("Copying contents") # cephadmlib is cephadm's private library of modules shutil.copytree( @@ -70,6 +115,7 @@ def _build(dest, src, versioning_vars=None): # cephadm.py is cephadm's main script for the "binary" # this must be renamed to __main__.py for the zipapp shutil.copy("cephadm.py", tempdir / "__main__.py") + versioning_vars = config.cli_args.version_vars if versioning_vars: generate_version_file(versioning_vars, tempdir / "_version.py") _compile(dest, tempdir) @@ -116,31 +162,77 @@ def _compile(dest, tempdir): log.info("Zipapp created without compression") -def _install_deps(tempdir): +def _install_deps(tempdir, config): """Install dependencies with pip.""" # TODO we could explicitly pass a python version here log.info("Installing dependencies") + + executable = sys.executable + venv = config.pip_venv + has_venv = _has_python_venv(sys.executable) if venv.enabled else False + venv = None + if venv == PipEnv.required and not has_venv: + raise RuntimeError('venv (virtual environment) module not found') + if has_venv: + log.info('Attempting to create a virtualenv') + venv = tempdir / "_venv_" + _run([sys.executable, '-m', 'venv', str(venv)]) + executable = str(venv / "bin" / pathlib.Path(executable).name) + # try to upgrade pip in the virtualenv. if it fails ignore the error + _run([executable, '-m', 'pip', 'install', '-U', 'pip']) + else: + log.info('Continuing without a virtualenv...') + if not _has_python_pip(executable): + raise RuntimeError('pip module not found') + # best effort to disable compilers, packages in the zipapp # must be pure python. env = os.environ.copy() env['CC'] = '/bin/false' env['CXX'] = '/bin/false' - # apparently pip doesn't have an API, just a cli. - subprocess.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - "--no-binary", - ":all:", - "--requirement", - "requirements.txt", - "--target", - tempdir, - ], - env=env, + if env.get('PYTHONPATH'): + env['PYTHONPATH'] = env['PYTHONPATH'] + f':{tempdir}' + else: + env['PYTHONPATH'] = f'{tempdir}' + if config.pip_split: + # a list of single item lists; so that pip run once for each + # requirement + req_batches = [[r] for r in config.requirements] + else: + # a list containing another list of the requirements, so we only + # need to run pip once + req_batches = [list(config.requirements)] + for batch in req_batches: + _run( + [ + executable, + "-m", + "pip", + "install", + "--no-binary", + ":all:", + "--target", + tempdir, + ] + batch, + env=env, + check=True, + ) + if venv: + shutil.rmtree(venv) + + +def _has_python_venv(executable): + res = _run( + [executable, '-m', 'venv', '--help'], stdout=subprocess.DEVNULL ) + return res.returncode == 0 + + +def _has_python_pip(executable): + res = _run( + [executable, '-m', 'venv', '--help'], stdout=subprocess.DEVNULL + ) + return res.returncode == 0 def generate_version_file(versioning_vars, dest): @@ -186,6 +278,12 @@ def main(): action="append", help="Set a key=value pair in the generated version info file", ) + parser.add_argument( + '--pip-use-venv', + choices=[e.name for e in PipEnv], + default=PipEnv.auto.name, + help='Configure pip to use a virtual environment when bundling dependencies', + ) args = parser.parse_args() if not _did_rexec() and args.python: @@ -196,7 +294,8 @@ def main(): v=sys.version_info ) ) - log.info("Args: %s", vars(args)) + for argkey, argval in vars(args).items(): + log.info("Argument: %s=%r", argkey, argval) if not HAS_ZIPAPP: # Unconditionally display an error that the version of python # lacks zipapp (probably too old). @@ -214,7 +313,7 @@ def main(): dest = pathlib.Path(args.dest).absolute() log.info("Source Dir: %s", source) log.info("Destination Path: %s", dest) - _build(dest, source, versioning_vars=args.version_vars) + _build(dest, source, Config(args)) if __name__ == "__main__": -- cgit v1.2.3