Merge pull request #54173 from phlogistonjohn/jjm-cephadm-jinja-2

cephadm: expand support for dependencies bundled into the zipapp

Reviewed-by: Adam King <adking@redhat.com>

1 files changed, 167 insertions, 10 deletions

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 122767a8ecc..1634eeac593 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -8,14 +8,15 @@
 import argparse
 import compileall
 import enum
+import json
 import logging
 import os
 import pathlib
+import shlex
 import shutil
 import subprocess
-import tempfile
-import shlex
 import sys
+import tempfile
 
 HAS_ZIPAPP = False
 try:
@@ -62,22 +63,70 @@ class PipEnv(enum.Enum):
         return self == self.auto or self == self.required
 
 
+class DependencyMode(enum.Enum):
+    pip = enum.auto()
+    rpm = enum.auto()
+    none = enum.auto()
+
+
 class Config:
     def __init__(self, cli_args):
         self.cli_args = cli_args
         self._maj_min = sys.version_info[0:2]
         self.install_dependencies = True
+        self.deps_mode = DependencyMode[cli_args.bundled_dependencies]
+        if self.deps_mode == DependencyMode.none:
+            self.install_dependencies = False
+        if self.deps_mode == DependencyMode.pip:
+            self._setup_pip()
+        elif self.deps_mode == DependencyMode.rpm:
+            self._setup_rpm()
+
+    def _setup_pip(self):
         if self._maj_min == (3, 6):
             self.pip_split = True
             self.requirements = PY36_REQUIREMENTS
         else:
             self.pip_split = False
             self.requirements = PY_REQUIREMENTS
-        self.pip_venv = PipEnv[cli_args.pip_use_venv]
+        self.pip_venv = PipEnv[self.cli_args.pip_use_venv]
+
+    def _setup_rpm(self):
+        self.requirements = [s.split()[0] for s in PY_REQUIREMENTS]
+
+
+class DependencyInfo:
+    """Type for tracking bundled dependencies."""
+
+    def __init__(self, config):
+        self._config = config
+        self._deps = []
+        self._reqs = {s.split()[0]: s for s in self._config.requirements}
+
+    @property
+    def requirements(self):
+        """Return requirements."""
+        return self._config.requirements
+
+    def add(self, name, **fields):
+        """Add a new bundled dependency to track."""
+        vals = {'name': name}
+        vals.update({k: v for k, v in fields.items() if v is not None})
+        if name in self._reqs:
+            vals['requirements_entry'] = self._reqs[name]
+        self._deps.append(vals)
+
+    def save(self, path):
+        """Record bundled dependency meta-data to the supplied file."""
+        with open(path, 'w') as fh:
+            json.dump(self._deps, fh)
+
 
 
 def _run(command, *args, **kwargs):
-    log.info('Running cmd: %s', ' '.join(shlex.quote(str(c)) for c in command))
+    log.info(
+        'Running cmd: %s', ' '.join(shlex.quote(str(c)) for c in command)
+    )
     return subprocess.run(command, *args, **kwargs)
 
 
@@ -104,9 +153,10 @@ def _build(dest, src, config):
     os.chdir(src)
     tempdir = pathlib.Path(tempfile.mkdtemp(suffix=".cephadm.build"))
     log.debug("working in %s", tempdir)
+    dinfo = None
     try:
         if config.install_dependencies:
-            _install_deps(tempdir, config)
+            dinfo = _install_deps(tempdir, config)
         log.info("Copying contents")
         # cephadmlib is cephadm's private library of modules
         shutil.copytree(
@@ -115,9 +165,14 @@ def _build(dest, src, config):
         # cephadm.py is cephadm's main script for the "binary"
         # this must be renamed to __main__.py for the zipapp
         shutil.copy("cephadm.py", tempdir / "__main__.py")
+        mdir = tempdir / "_cephadmmeta"
+        mdir.mkdir(parents=True, exist_ok=True)
+        (mdir / "__init__.py").touch(exist_ok=True)
         versioning_vars = config.cli_args.version_vars
         if versioning_vars:
-            generate_version_file(versioning_vars, tempdir / "_version.py")
+            generate_version_file(versioning_vars, mdir / "version.py")
+        if dinfo:
+            dinfo.save(mdir / "deps.json")
         _compile(dest, tempdir)
     finally:
         shutil.rmtree(tempdir)
@@ -128,7 +183,9 @@ def _ignore_cephadmlib(source_dir, names):
     return [
         name
         for name in names
-        if name.endswith(("~", ".old", ".swp", ".pyc", ".pyo", "__pycache__"))
+        if name.endswith(
+            ("~", ".old", ".swp", ".pyc", ".pyo", ".so", "__pycache__")
+        )
     ]
 
 
@@ -163,9 +220,16 @@ def _compile(dest, tempdir):
 
 
 def _install_deps(tempdir, config):
+    if config.deps_mode == DependencyMode.pip:
+        return _install_pip_deps(tempdir, config)
+    if config.deps_mode == DependencyMode.rpm:
+        return _install_rpm_deps(tempdir, config)
+    raise ValueError(f'unexpected deps mode: {deps.mode}')
+
+
+def _install_pip_deps(tempdir, config):
     """Install dependencies with pip."""
-    # TODO we could explicitly pass a python version here
-    log.info("Installing dependencies")
+    log.info("Installing dependencies using pip")
 
     executable = sys.executable
     venv = config.pip_venv
@@ -214,12 +278,29 @@ def _install_deps(tempdir, config):
                 ":all:",
                 "--target",
                 tempdir,
-            ] + batch,
+            ]
+            + batch,
             env=env,
             check=True,
         )
+
+    dinfo = DependencyInfo(config)
+    res = _run(
+        [executable, '-m', 'pip', 'list', '--format=json', '--path', tempdir],
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+    pkgs = json.loads(res.stdout)
+    for pkg in pkgs:
+        dinfo.add(
+            pkg['name'],
+            version=pkg['version'],
+            package_source='pip',
+        )
+
     if venv:
         shutil.rmtree(venv)
+    return dinfo
 
 
 def _has_python_venv(executable):
@@ -236,6 +317,75 @@ def _has_python_pip(executable):
     return res.returncode == 0
 
 
+def _install_rpm_deps(tempdir, config):
+    log.info("Installing dependencies using RPMs")
+    dinfo = DependencyInfo(config)
+    for pkg in config.requirements:
+        log.info(f"Looking for rpm package for: {pkg!r}")
+        _deps_from_rpm(tempdir, config, dinfo, pkg)
+    return dinfo
+
+
+def _deps_from_rpm(tempdir, config, dinfo, pkg):
+    # first, figure out what rpm provides a particular python lib
+    dist = f'python3dist({pkg})'.lower()
+    try:
+        res = subprocess.run(
+            ['rpm', '-q', '--whatprovides', dist],
+            check=True,
+            stdout=subprocess.PIPE,
+        )
+    except subprocess.CalledProcessError as err:
+        log.error(f"Command failed: {err.args[1]!r}")
+        log.error(f"An installed RPM package for {pkg} was not found")
+        sys.exit(1)
+    rpmname = res.stdout.strip().decode('utf8')
+    # get version information about said rpm
+    res = subprocess.run(
+        ['rpm', '-q', '--qf', '%{version} %{release} %{epoch}\\n', rpmname],
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+    vers = res.stdout.decode('utf8').splitlines()[0].split()
+    log.info(f"RPM Package: {rpmname} ({vers})")
+    dinfo.add(
+        pkg,
+        rpm_name=rpmname,
+        version=vers[0],
+        rpm_release=vers[1],
+        rpm_epoch=vers[2],
+        package_source='rpm',
+    )
+    # get the list of files provided by the rpm
+    res = subprocess.run(
+        ['rpm', '-ql', rpmname], check=True, stdout=subprocess.PIPE
+    )
+    paths = [l.decode('utf8') for l in res.stdout.splitlines()]
+    # the top_level.txt file can be used to determine where the python packages
+    # actually are. We need all of those and the meta-data dir (parent of
+    # top_level.txt) to be included in our zipapp
+    top_level = None
+    for path in paths:
+        if path.endswith('top_level.txt'):
+            top_level = pathlib.Path(path)
+    if not top_level:
+        raise ValueError('top_level not found')
+    meta_dir = top_level.parent
+    pkg_dirs = [
+        top_level.parent.parent / p
+        for p in top_level.read_text().splitlines()
+    ]
+    meta_dest = tempdir / meta_dir.name
+    log.info(f"Copying {meta_dir} to {meta_dest}")
+    # copy the meta data directory
+    shutil.copytree(meta_dir, meta_dest, ignore=_ignore_cephadmlib)
+    # copy all the package directories
+    for pkg_dir in pkg_dirs:
+        pkg_dest = tempdir / pkg_dir.name
+        log.info(f"Copying {pkg_dir} to {pkg_dest}")
+        shutil.copytree(pkg_dir, pkg_dest, ignore=_ignore_cephadmlib)
+
+
 def generate_version_file(versioning_vars, dest):
     log.info("Generating version file")
     log.debug("versioning_vars=%r", versioning_vars)
@@ -285,6 +435,13 @@ def main():
         default=PipEnv.auto.name,
         help='Configure pip to use a virtual environment when bundling dependencies',
     )
+    parser.add_argument(
+        "--bundled-dependencies",
+        "-B",
+        choices=[e.name for e in DependencyMode],
+        default=DependencyMode.pip.name,
+        help="Source for bundled dependencies",
+    )
     args = parser.parse_args()
 
     if not _did_rexec() and args.python: