cephadm: add modes for bundled deps and track what is installed

Add additional modes for how to handle bundled dependencies.
Add a mode to disable bundled dependencies entirely.
Add a mode to source dependencies from installed RPM packages. This mode
will likely be the basis for some distributions of ceph were the exact
version of the package can be used via a controlled build environment.

Add the capability to store metadata about the bundled dependencies
in the zipapp. Move all the metadata created by build.py to a synthetic
package `_cephadmmeta` that gets created when building the zipapp.

Signed-off-by: John Mulligan <jmulligan@redhat.com>

1 files changed, 167 insertions, 10 deletions

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 87d990e72a3..1cd70d6da92 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -8,14 +8,15 @@
 import argparse
 import compileall
 import enum
+import json
 import logging
 import os
 import pathlib
+import shlex
 import shutil
 import subprocess
-import tempfile
-import shlex
 import sys
+import tempfile
 
 HAS_ZIPAPP = False
 try:
@@ -62,22 +63,70 @@ class PipEnv(enum.Enum):
         return self == self.auto or self == self.required
 
 
+class DependencyMode(enum.Enum):
+    pip = enum.auto()
+    rpm = enum.auto()
+    none = enum.auto()
+
+
 class Config:
     def __init__(self, cli_args):
         self.cli_args = cli_args
         self._maj_min = sys.version_info[0:2]
         self.install_dependencies = True
+        self.deps_mode = DependencyMode[cli_args.bundled_dependencies]
+        if self.deps_mode == DependencyMode.none:
+            self.install_dependencies = False
+        if self.deps_mode == DependencyMode.pip:
+            self._setup_pip()
+        elif self.deps_mode == DependencyMode.rpm:
+            self._setup_rpm()
+
+    def _setup_pip(self):
         if self._maj_min == (3, 6):
             self.pip_split = True
             self.requirements = PY36_REQUIREMENTS
         else:
             self.pip_split = False
             self.requirements = PY_REQUIREMENTS
-        self.pip_venv = PipEnv[cli_args.pip_use_venv]
+        self.pip_venv = PipEnv[self.cli_args.pip_use_venv]
+
+    def _setup_rpm(self):
+        self.requirements = [s.split()[0] for s in PY_REQUIREMENTS]
+
+
+class DependencyInfo:
+    """Type for tracking bundled dependencies."""
+
+    def __init__(self, config):
+        self._config = config
+        self._deps = []
+        self._reqs = {s.split()[0]: s for s in self._config.requirements}
+
+    @property
+    def requirements(self):
+        """Return requirements."""
+        return self._config.requirements
+
+    def add(self, name, **fields):
+        """Add a new bundled dependency to track."""
+        vals = {'name': name}
+        vals.update({k: v for k, v in fields.items() if v is not None})
+        if name in self._reqs:
+            vals['requirements_entry'] = self._reqs[name]
+        self._deps.append(vals)
+
+    def save(self, path):
+        """Record bundled dependency meta-data to the supplied file."""
+        with open(path, 'w') as fh:
+            json.dump(self._deps, fh)
+
 
 
 def _run(command, *args, **kwargs):
-    log.info('Running cmd: %s', ' '.join(shlex.quote(str(c)) for c in command))
+    log.info(
+        'Running cmd: %s', ' '.join(shlex.quote(str(c)) for c in command)
+    )
     return subprocess.run(command, *args, **kwargs)
 
 
@@ -104,9 +153,10 @@ def _build(dest, src, config):
     os.chdir(src)
     tempdir = pathlib.Path(tempfile.mkdtemp(suffix=".cephadm.build"))
     log.debug("working in %s", tempdir)
+    dinfo = None
     try:
         if config.install_dependencies:
-            _install_deps(tempdir, config)
+            dinfo = _install_deps(tempdir, config)
         log.info("Copying contents")
         # cephadmlib is cephadm's private library of modules
         shutil.copytree(
@@ -115,9 +165,14 @@ def _build(dest, src, config):
         # cephadm.py is cephadm's main script for the "binary"
         # this must be renamed to __main__.py for the zipapp
         shutil.copy("cephadm.py", tempdir / "__main__.py")
+        mdir = tempdir / "_cephadmmeta"
+        mdir.mkdir(parents=True, exist_ok=True)
+        (mdir / "__init__.py").touch(exist_ok=True)
         versioning_vars = config.cli_args.version_vars
         if versioning_vars:
-            generate_version_file(versioning_vars, tempdir / "_version.py")
+            generate_version_file(versioning_vars, mdir / "version.py")
+        if dinfo:
+            dinfo.save(mdir / "deps.json")
         _compile(dest, tempdir)
     finally:
         shutil.rmtree(tempdir)
@@ -128,7 +183,9 @@ def _ignore_cephadmlib(source_dir, names):
     return [
         name
         for name in names
-        if name.endswith(("~", ".old", ".swp", ".pyc", ".pyo", "__pycache__"))
+        if name.endswith(
+            ("~", ".old", ".swp", ".pyc", ".pyo", ".so", "__pycache__")
+        )
     ]
 
 
@@ -163,9 +220,16 @@ def _compile(dest, tempdir):
 
 
 def _install_deps(tempdir, config):
+    if config.deps_mode == DependencyMode.pip:
+        return _install_pip_deps(tempdir, config)
+    if config.deps_mode == DependencyMode.rpm:
+        return _install_rpm_deps(tempdir, config)
+    raise ValueError(f'unexpected deps mode: {deps.mode}')
+
+
+def _install_pip_deps(tempdir, config):
     """Install dependencies with pip."""
-    # TODO we could explicitly pass a python version here
-    log.info("Installing dependencies")
+    log.info("Installing dependencies using pip")
 
     executable = sys.executable
     venv = config.pip_venv
@@ -213,12 +277,29 @@ def _install_deps(tempdir, config):
                 ":all:",
                 "--target",
                 tempdir,
-            ] + batch,
+            ]
+            + batch,
             env=env,
             check=True,
         )
+
+    dinfo = DependencyInfo(config)
+    res = _run(
+        [executable, '-m', 'pip', 'list', '--format=json', '--path', tempdir],
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+    pkgs = json.loads(res.stdout)
+    for pkg in pkgs:
+        dinfo.add(
+            pkg['name'],
+            version=pkg['version'],
+            package_source='pip',
+        )
+
     if venv:
         shutil.rmtree(venv)
+    return dinfo
 
 
 def _has_python_venv(executable):
@@ -235,6 +316,75 @@ def _has_python_pip(executable):
     return res.returncode == 0
 
 
+def _install_rpm_deps(tempdir, config):
+    log.info("Installing dependencies using RPMs")
+    dinfo = DependencyInfo(config)
+    for pkg in config.requirements:
+        log.info(f"Looking for rpm package for: {pkg!r}")
+        _deps_from_rpm(tempdir, config, dinfo, pkg)
+    return dinfo
+
+
+def _deps_from_rpm(tempdir, config, dinfo, pkg):
+    # first, figure out what rpm provides a particular python lib
+    dist = f'python3dist({pkg})'.lower()
+    try:
+        res = subprocess.run(
+            ['rpm', '-q', '--whatprovides', dist],
+            check=True,
+            stdout=subprocess.PIPE,
+        )
+    except subprocess.CalledProcessError as err:
+        log.error(f"Command failed: {err.args[1]!r}")
+        log.error(f"An installed RPM package for {pkg} was not found")
+        sys.exit(1)
+    rpmname = res.stdout.strip().decode('utf8')
+    # get version information about said rpm
+    res = subprocess.run(
+        ['rpm', '-q', '--qf', '%{version} %{release} %{epoch}\\n', rpmname],
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+    vers = res.stdout.decode('utf8').splitlines()[0].split()
+    log.info(f"RPM Package: {rpmname} ({vers})")
+    dinfo.add(
+        pkg,
+        rpm_name=rpmname,
+        version=vers[0],
+        rpm_release=vers[1],
+        rpm_epoch=vers[2],
+        package_source='rpm',
+    )
+    # get the list of files provided by the rpm
+    res = subprocess.run(
+        ['rpm', '-ql', rpmname], check=True, stdout=subprocess.PIPE
+    )
+    paths = [l.decode('utf8') for l in res.stdout.splitlines()]
+    # the top_level.txt file can be used to determine where the python packages
+    # actually are. We need all of those and the meta-data dir (parent of
+    # top_level.txt) to be included in our zipapp
+    top_level = None
+    for path in paths:
+        if path.endswith('top_level.txt'):
+            top_level = pathlib.Path(path)
+    if not top_level:
+        raise ValueError('top_level not found')
+    meta_dir = top_level.parent
+    pkg_dirs = [
+        top_level.parent.parent / p
+        for p in top_level.read_text().splitlines()
+    ]
+    meta_dest = tempdir / meta_dir.name
+    log.info(f"Copying {meta_dir} to {meta_dest}")
+    # copy the meta data directory
+    shutil.copytree(meta_dir, meta_dest, ignore=_ignore_cephadmlib)
+    # copy all the package directories
+    for pkg_dir in pkg_dirs:
+        pkg_dest = tempdir / pkg_dir.name
+        log.info(f"Copying {pkg_dir} to {pkg_dest}")
+        shutil.copytree(pkg_dir, pkg_dest, ignore=_ignore_cephadmlib)
+
+
 def generate_version_file(versioning_vars, dest):
     log.info("Generating version file")
     log.debug("versioning_vars=%r", versioning_vars)
@@ -284,6 +434,13 @@ def main():
         default=PipEnv.auto.name,
         help='Configure pip to use a virtual environment when bundling dependencies',
     )
+    parser.add_argument(
+        "--bundled-dependencies",
+        "-B",
+        choices=[e.name for e in DependencyMode],
+        default=DependencyMode.pip.name,
+        help="Source for bundled dependencies",
+    )
     args = parser.parse_args()
 
     if not _did_rexec() and args.python: