diff options
40 files changed, 279 insertions, 282 deletions
@@ -1,3 +1,4 @@ +**/__pycache__/ *.6 *.Plo *.a @@ -8,6 +9,7 @@ *.gcno *.gcov *.info +*.junit.xml *.la *.lo *.log @@ -18,12 +20,15 @@ *.so.* *.swp *~ +.coverage .deps .dirstamp .libs +.mypy_cache .pytest_cache -/.build-depend +/.build*/ /.cache +/.install_dev /aclocal.m4 /ar-lib /autom4te.cache/* @@ -44,6 +49,7 @@ /daemon/lua/*.inc /daemon/lua/trust_anchors.lua /depcomp +/dist /distro/tests/*/.vagrant /doc/**/.doctrees /doc/**/doxyxml @@ -82,11 +88,4 @@ _obj kresd.amalg.c libkres.amalg.c luacov.*.out -**/__pycache__/ -dist/ -.coverage -.mypy_cache -.install_kresd/ -.build_kresd/ poetry.lock -*.junit.xml diff --git a/.gitlab-ci.manager.yml b/.gitlab-ci.manager.yml index cc94c0bf..727e1bf1 100644 --- a/.gitlab-ci.manager.yml +++ b/.gitlab-ci.manager.yml @@ -31,8 +31,6 @@ lint:py3.12: stage: check script: - poetry install --only main,dev,test - # create required directories that are in default config, otherwise unit tests fail - - mkdir -p /var/cache/knot-resolver - poe test # the following command makes sure that the source root of the coverage file is at $gitroot - poetry run bash -c "coverage combine .coverage; coverage xml" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0caf1471..1415c34e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,7 @@ variables: # IMAGE_TAG is a Git branch/tag name from https://gitlab.nic.cz/knot/knot-resolver-ci # In general, keep it pointing to a tag - use a branch only for development. # More info in the knot-resolver-ci repository. - IMAGE_TAG: 'v20240828' + IMAGE_TAG: 'v20240924' IMAGE_PREFIX: '$CI_REGISTRY/knot/knot-resolver-ci' image: $IMAGE_PREFIX/debian12-knot_3_3:$IMAGE_TAG @@ -688,10 +688,14 @@ pkg:alma-9: pkg:arch: <<: *pkg_test_user - image: $CI_REGISTRY/packaging/apkg/full/arch - allow_failure: true # our image not starting up currently + image: $CI_REGISTRY/packaging/apkg/test/arch + tags: + - docker + - linux + - amd64 before_script: - pacman -Syy + - pip install apkg # RHEL 8 derivatives would need more work due to *default* python being old #pkg:rocky-8: diff --git a/ci/images/manager/Dockerfile b/ci/images/manager/Dockerfile deleted file mode 100644 index 69ed251e..00000000 --- a/ci/images/manager/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -FROM fedora:38 -LABEL Knot Resolver <knot-resolver@labs.nic.cz> - -WORKDIR /root -CMD ["/bin/bash"] -ENV PATH="/root/.local/bin:${PATH}" - -# Install Python and deps -RUN dnf install -y\ - python3.8 python3.9 python3.10 python3.10-devel\ - python3.11 python3.11-devel python3.12 python3.12-devel\ - python3-gobject pipx git which diffutils gcc pkg-config\ - cairo-devel gobject-introspection-devel cairo-gobject-devel\ - && dnf clean all - -# Install poetry -RUN pipx install poetry==1.4.2\ - # not exactly required, but helpful - && pipx install poethepoet diff --git a/distro/pkg/arch/PKGBUILD b/distro/pkg/arch/PKGBUILD index 6f473ca5..ebbd164d 100644 --- a/distro/pkg/arch/PKGBUILD +++ b/distro/pkg/arch/PKGBUILD @@ -63,9 +63,6 @@ build() { -D malloc=jemalloc \ -D unit_tests=enabled ninja -C build - pushd build/python - python -Pm build --wheel --no-isolation - popd python -Pm build --wheel --no-isolation } @@ -85,13 +82,8 @@ package() { # remove modules with missing dependencies rm "${pkgdir}/usr/lib/knot-resolver/kres_modules/etcd.lua" - # install knot-resolver metadata Python module - pushd build/python + # install knot-resolver Python module python -Pm installer --destdir="$pkgdir" dist/*.whl - popd - # install knot-resolver-manager - python -Pm installer --destdir="$pkgdir" dist/*.whl install -m 644 -D etc/config/config.yaml ${pkgdir}/etc/knot-resolver/config.yaml - } diff --git a/distro/pkg/deb/rules b/distro/pkg/deb/rules index a682d79b..66c71c3e 100755 --- a/distro/pkg/deb/rules +++ b/distro/pkg/deb/rules @@ -35,17 +35,11 @@ override_dh_auto_build: -Dc_args="$${CFLAGS}" \ -Dc_link_args="$${LDFLAGS}" ninja -v -C build_deb - PYBUILD_NAME=knot_resolver PYBUILD_DESTDIR="$${PYKRES_DESTDIR}" \ - dh_auto_build --buildsystem=pybuild - PYBUILD_NAME=knot_resolver_build_options PYBUILD_DESTDIR="$${PYKRES_DESTDIR}" \ - dh_auto_build --buildsystem=pybuild --sourcedirectory build_deb/python + PYBUILD_NAME=knot_resolver PYBUILD_DESTDIR="$${PYKRES_DESTDIR}" dh_auto_build override_dh_auto_install: DESTDIR="$(shell pwd)/debian/tmp" ninja -v -C build_deb install - PYBUILD_NAME=knot_resolver PYBUILD_DESTDIR="$${PYKRES_DESTDIR}" \ - dh_auto_install --buildsystem=pybuild - PYBUILD_NAME=knot_resolver_build_options PYBUILD_DESTDIR="$${PYKRES_DESTDIR}" \ - dh_auto_install --buildsystem=pybuild --sourcedirectory build_deb/python + PYBUILD_NAME=knot_resolver PYBUILD_DESTDIR="$${PYKRES_DESTDIR}" dh_auto_install install -m 644 -D etc/config/config.yaml debian/tmp/etc/knot-resolver/config.yaml override_dh_auto_test: diff --git a/distro/pkg/rpm/knot-resolver.spec b/distro/pkg/rpm/knot-resolver.spec index d1ec860c..ca8602ff 100644 --- a/distro/pkg/rpm/knot-resolver.spec +++ b/distro/pkg/rpm/knot-resolver.spec @@ -181,10 +181,6 @@ CFLAGS="%{optflags}" LDFLAGS="%{?__global_ldflags}" meson build_rpm \ %{NINJA} -v -C build_rpm -pushd build_rpm/python -%py3_build -popd - %py3_build %install @@ -216,11 +212,6 @@ mv %{buildroot}/%{_datadir}/doc/%{name}/* %{buildroot}/%{_pkgdocdir}/ # install knot_resolver python module %py3_install -# install knot_resolver_build_options python module -pushd build_rpm/python -%py3_install -popd - install -m 644 -D etc/config/config.yaml %{buildroot}%{_sysconfdir}/knot-resolver/config.yaml %pre @@ -301,14 +292,7 @@ getent passwd knot-resolver >/dev/null || useradd -r -g knot-resolver -d %{_sysc %{_libdir}/knot-resolver/kres_modules/view.lua %{_libdir}/knot-resolver/kres_modules/watchdog.lua %{_libdir}/knot-resolver/kres_modules/workarounds.lua -%{python3_sitelib}/knot_resolver_build_options.py -%{python3_sitelib}/knot_resolver_build_options-* %{python3_sitearch}/knot_resolver* -%if 0%{?suse_version} -%pycache_only %{python3_sitelib}/__pycache__/knot_resolver_build_options.* -%else -%{python3_sitelib}/__pycache__/knot_resolver_build_options.* -%endif %{_mandir}/man8/kresd.8.gz %{_mandir}/man8/kresctl.8.gz diff --git a/doc/_static/config.schema.json b/doc/_static/config.schema.json index 989aaf62..036a3ce0 100644 --- a/doc/_static/config.schema.json +++ b/doc/_static/config.schema.json @@ -29,7 +29,7 @@ "rundir": { "type": "string", "description": "Directory where the resolver can create files and which will be it's cwd.", - "default": "/var/run/knot-resolver" + "default": "/run/knot-resolver" }, "workers": { "anyOf": [ @@ -75,7 +75,7 @@ } }, "default": { - "unix_socket": "/var/run/knot-resolver/kres-api.sock", + "unix_socket": "/run/knot-resolver/kres-api.sock", "interface": null } }, diff --git a/etc/config/config.dev.yaml b/etc/config/config.dev.yaml index 1ceddfb3..6705069e 100644 --- a/etc/config/config.dev.yaml +++ b/etc/config/config.dev.yaml @@ -1,9 +1,6 @@ -rundir: ./runtime workers: 1 management: interface: 127.0.0.1@5000 -cache: - storage: ./cache logging: level: notice groups: diff --git a/meson.build b/meson.build index 0bd46ad6..82035cec 100644 --- a/meson.build +++ b/meson.build @@ -33,6 +33,7 @@ message('------------------------------') # Variables +auto_prefixes = ['/', '/usr', '/usr/local'] libkres_soversion = 9 libext = '.so' @@ -54,7 +55,15 @@ modules_dir = lib_dir / 'kres_modules' sbin_dir = prefix / get_option('sbindir') bin_dir = prefix / get_option('bindir') if host_machine.system() == 'linux' - run_dir = '/run' / 'knot-resolver' + # When installing from sources with a non-standard prefix, + # we need to set the correct run directory with the prefix, + # otherwise rwx permissions will fail with a validation error + # on the run directory + if prefix in auto_prefixes + run_dir = '/run' / 'knot-resolver' + else + run_dir = prefix / 'run' / 'knot-resolver' + endif elif host_machine.system() == 'darwin' run_dir = prefix / get_option('localstatedir') / 'run' / 'knot-resolver' else @@ -71,7 +80,6 @@ completion_dir = prefix / 'share' # When installing from sources into a non-standard prefix and the library is # shared/dynamic, we need to set the executables' RPATH so that they can find # `libkresd`, otherwise running them will fail with dynamic linkage errors -auto_prefixes = ['/', '/usr', '/usr/local'] rpath_opt = get_option('install_rpath') if (get_option('default_library') == 'static' or rpath_opt == 'disabled' or @@ -267,7 +275,7 @@ subdir('lib') ## Remaining code subdir('daemon') subdir('modules') -subdir('python') +subdir('python' / 'knot_resolver') subdir('utils') if get_option('bench') == 'enabled' subdir('bench') diff --git a/pyproject.toml b/pyproject.toml index 4c20d298..e545eaa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ documentation = "https://www.knot-resolver.cz/documentation" packages = [ {include = "knot_resolver", from = "python"} ] +exclude = ["**/*.in", "**/meson.build"] # See currently open issue about building C extensions here: # https://github.com/python-poetry/poetry/issues/2740 @@ -74,11 +75,12 @@ knot-resolver = 'knot_resolver.manager.main:main' configure = { cmd = "scripts/poe-tasks/configure", help = "(Re)configure Meson build directory" } run = { cmd = "scripts/poe-tasks/run", help = "Run Knot Resolver" } run-debug = { cmd = "scripts/poe-tasks/run-debug", help = "Debug Knot Resolver with debugpy" } -docs = { cmd = "scripts/poe-tasks/docs", help = "Create Knot Resolver HTML documentation" } +doc = { cmd = "scripts/poe-tasks/doc", help = "Create Knot Resolver HTML documentation" } +doc-schema = { cmd = "scripts/poe-tasks/doc-schema", help = "Generate a JSON schema of the Knot Resolver configuration"} test = { cmd = "scripts/poe-tasks/test", help = "Run pytest unit tests" } check = { cmd = "scripts/poe-tasks/check", help = "Check that all dependencies are installed and run static code analysis" } examples = { cmd = "scripts/poe-tasks/examples", help = "Validate all configuration examples using 'kresctl validate' utility" } -man = {cmd = "scripts/poe-tasks/man", help = "Display Knot Resolver manpage from sources" } +gen-constantspy = { cmd = "scripts/poe-tasks/gen-constantspy", help = "Generate 'constants.py' module using Meson configured options" } gen-setuppy = { cmd = "scripts/poe-tasks/gen-setuppy", help = "Generate 'setup.py' file for backwards compatibility" } format = { cmd = "scripts/poe-tasks/format", help = "Run code formatter" } kresctl = { cmd = "scripts/poe-tasks/kresctl", help="Run kresctl utility" } diff --git a/python/knot_resolver/client/command.py b/python/knot_resolver/client/command.py index da42f2cf..960ac1f5 100644 --- a/python/knot_resolver/client/command.py +++ b/python/knot_resolver/client/command.py @@ -1,16 +1,10 @@ import argparse -import os from abc import ABC, abstractmethod # pylint: disable=[no-name-in-module] from pathlib import Path from typing import Dict, List, Optional, Tuple, Type, TypeVar from urllib.parse import quote -from knot_resolver.constants import ( - API_SOCK_PATH_DEFAULT, - API_SOCK_PATH_ENV_VAR, - CONFIG_FILE_PATH_DEFAULT, - CONFIG_FILE_PATH_ENV_VAR, -) +from knot_resolver.constants import API_SOCK_FILE, CONFIG_FILE from knot_resolver.datamodel.types import IPAddressPort from knot_resolver.utils.modeling import parsing from knot_resolver.utils.modeling.exceptions import DataValidationError @@ -74,27 +68,18 @@ def determine_socket(namespace: argparse.Namespace) -> SocketDesc: if len(namespace.socket) > 0: return SocketDesc(namespace.socket[0], "--socket argument") - config_path = os.getenv(CONFIG_FILE_PATH_ENV_VAR) - socket_env = os.getenv(API_SOCK_PATH_ENV_VAR) - socket: Optional[SocketDesc] = None # 2) socket from config file ('--config' argument) if len(namespace.config) > 0: socket = get_socket_from_config(namespace.config[0], False) - # 3) socket from config file (environment variable) - elif config_path: - socket = get_socket_from_config(Path(config_path), False) - # 4) socket from environment variable - elif socket_env: - socket = SocketDesc(socket_env, f'Environment variable "{API_SOCK_PATH_ENV_VAR}"') - # 5) socket from config file (default config file constant) + # 3) socket from config file (default config file constant) else: - socket = get_socket_from_config(CONFIG_FILE_PATH_DEFAULT, True) + socket = get_socket_from_config(CONFIG_FILE, True) if socket: return socket - # 6) socket default - return SocketDesc(str(API_SOCK_PATH_DEFAULT), f'Default value "{API_SOCK_PATH_DEFAULT}"') + # 4) socket default + return SocketDesc(str(API_SOCK_FILE), f'Default value "{API_SOCK_FILE}"') class CommandArgs: diff --git a/python/knot_resolver/constants.py b/python/knot_resolver/constants.py index f63478d4..2acb8660 100644 --- a/python/knot_resolver/constants.py +++ b/python/knot_resolver/constants.py @@ -1,40 +1,19 @@ -import logging -from importlib.metadata import version -from importlib.util import find_spec from pathlib import Path -# Installed Knot Resolver build options from Meson is semi-optional. -# They are needed to run the resolver, but not for its unit tests. -if find_spec("knot_resolver_build_options"): - import knot_resolver_build_options as build_conf # type: ignore[import-not-found] -else: - build_conf = None +VERSION = "6.0.8" +USER = "knot-resolver" +GROUP = "knot-resolver" -VERSION = version("knot_resolver") if find_spec("knot_resolver") else "6" -WORKERS_MAX_DEFAULT = 256 -LOGGING_LEVEL_STARTUP = logging.DEBUG -PID_FILE_NAME = "knot-resolver.pid" +# dirs paths +RUN_DIR = Path("/run/knot-resolver") +ETC_DIR = Path("/etc/knot-resolver") +SBIN_DIR = Path("/usr/sbin") +CACHE_DIR = Path("/var/cache/knot-resolver") -FIX_COUNTER_ATTEMPTS_MAX = 2 -FIX_COUNTER_DECREASE_INTERVAL_SEC = 30 * 60 -WATCHDOG_INTERVAL_SEC: float = 5 +# files paths +CONFIG_FILE = ETC_DIR / "config.yaml" +API_SOCK_FILE = RUN_DIR / "kres-api.sock" -USER_DEFAULT = build_conf.user if build_conf else "knot-resolver" -GROUP_DEFAULT = build_conf.group if build_conf else "knot-resolver" - -RUN_DIR_DEFAULT: Path = build_conf.run_dir if build_conf else Path("/var/run/knot-resolver") -ETC_DIR_DEFAULT: Path = build_conf.etc_dir if build_conf else Path("/etc/knot-resolver") -CONFIG_FILE_PATH_DEFAULT = ETC_DIR_DEFAULT / "config.yaml" -CONFIG_FILE_PATH_ENV_VAR = "KRES_MANAGER_CONFIG" -API_SOCK_PATH_DEFAULT = RUN_DIR_DEFAULT / "kres-api.sock" -API_SOCK_PATH_ENV_VAR = "KRES_MANAGER_API_SOCK" - - -def kresd_executable() -> Path: - assert build_conf is not None - return build_conf.sbin_dir / "kresd" - - -def kres_cache_gc_executable() -> Path: - assert build_conf is not None - return build_conf.sbin_dir / "kres-cache-gc" +# executables paths +KRESD_EXECUTABLE = SBIN_DIR / "kresd" +KRES_CACHE_GC_EXECUTABLE = SBIN_DIR / "kres-cache-gc" diff --git a/python/knot_resolver/constants.py.in b/python/knot_resolver/constants.py.in new file mode 100644 index 00000000..0f1c3a88 --- /dev/null +++ b/python/knot_resolver/constants.py.in @@ -0,0 +1,19 @@ +from pathlib import Path + +VERSION = "@version@" +USER = "@user@" +GROUP = "@group@" + +# dirs paths +RUN_DIR = Path("@run_dir@") +ETC_DIR = Path("@etc_dir@") +SBIN_DIR = Path("@sbin_dir@") +CACHE_DIR = Path("@cache_dir@") + +# files paths +CONFIG_FILE = ETC_DIR / "config.yaml" +API_SOCK_FILE = RUN_DIR / "kres-api.sock" + +# executables paths +KRESD_EXECUTABLE = SBIN_DIR / "kresd" +KRES_CACHE_GC_EXECUTABLE = SBIN_DIR / "kres-cache-gc" diff --git a/python/knot_resolver/controller/supervisord/config_file.py b/python/knot_resolver/controller/supervisord/config_file.py index a9144505..45a1a83e 100644 --- a/python/knot_resolver/controller/supervisord/config_file.py +++ b/python/knot_resolver/controller/supervisord/config_file.py @@ -6,7 +6,7 @@ from typing import Literal from jinja2 import Template -from knot_resolver.constants import kres_cache_gc_executable, kresd_executable +from knot_resolver.constants import KRES_CACHE_GC_EXECUTABLE, KRESD_EXECUTABLE from knot_resolver.controller.interface import KresID, SubprocessType from knot_resolver.datamodel.config_schema import KresConfig from knot_resolver.datamodel.logging_schema import LogTargetEnum @@ -95,7 +95,7 @@ class ProcessTypeConfig: return ProcessTypeConfig( # type: ignore[call-arg] logfile=supervisord_subprocess_log_dir(config) / "gc.log", workdir=cwd, - command=f"{kres_cache_gc_executable()} -c {kres_cache_dir(config)}{kres_cache_gc_args(config)}", + command=f"{KRES_CACHE_GC_EXECUTABLE} -c {kres_cache_dir(config)}{kres_cache_gc_args(config)}", environment="", ) @@ -105,7 +105,7 @@ class ProcessTypeConfig: return ProcessTypeConfig( # type: ignore[call-arg] logfile=supervisord_subprocess_log_dir(config) / "policy-loader.log", workdir=cwd, - command=f"{kresd_executable()} -c {(policy_loader_config_file(config))} -c - -n", + command=f"{KRESD_EXECUTABLE} -c {(policy_loader_config_file(config))} -c - -n", environment="X-SUPERVISORD-TYPE=notify", ) @@ -115,7 +115,7 @@ class ProcessTypeConfig: return ProcessTypeConfig( # type: ignore[call-arg] logfile=supervisord_subprocess_log_dir(config) / "kresd%(process_num)d.log", workdir=cwd, - command=f"{kresd_executable()} -c {kresd_config_file_supervisord_pattern(config)} -n", + command=f"{KRESD_EXECUTABLE} -c {kresd_config_file_supervisord_pattern(config)} -n", environment='SYSTEMD_INSTANCE="%(process_num)d",X-SUPERVISORD-TYPE=notify', max_procs=int(config.max_workers) + 1, # +1 for the canary process ) diff --git a/python/knot_resolver/datamodel/cache_schema.py b/python/knot_resolver/datamodel/cache_schema.py index 3f7d1dc2..d40ee2a0 100644 --- a/python/knot_resolver/datamodel/cache_schema.py +++ b/python/knot_resolver/datamodel/cache_schema.py @@ -1,5 +1,6 @@ from typing import List, Literal, Optional, Union +from knot_resolver.constants import CACHE_DIR from knot_resolver.datamodel.templates import template_from_str from knot_resolver.datamodel.types import ( DNSRecordTypeEnum, @@ -123,7 +124,7 @@ class CacheSchema(ConfigSchema): prefetch: These options help keep the cache hot by prefetching expiring records or learning usage patterns and repetitive queries. """ - storage: WritableDir = lazy_default(WritableDir, "/var/cache/knot-resolver") + storage: WritableDir = lazy_default(WritableDir, str(CACHE_DIR)) size_max: SizeUnit = SizeUnit("100M") garbage_collector: Union[GarbageCollectorSchema, Literal[False]] = GarbageCollectorSchema() ttl_min: TimeUnit = TimeUnit("5s") diff --git a/python/knot_resolver/datamodel/config_schema.py b/python/knot_resolver/datamodel/config_schema.py index fe18516b..ddadd7fb 100644 --- a/python/knot_resolver/datamodel/config_schema.py +++ b/python/knot_resolver/datamodel/config_schema.py @@ -3,7 +3,7 @@ import os import socket from typing import Any, Dict, List, Literal, Optional, Tuple, Union -from knot_resolver.constants import API_SOCK_PATH_DEFAULT, RUN_DIR_DEFAULT, VERSION, WORKERS_MAX_DEFAULT +from knot_resolver.constants import API_SOCK_FILE, RUN_DIR, VERSION from knot_resolver.datamodel.cache_schema import CacheSchema from knot_resolver.datamodel.dns64_schema import Dns64Schema from knot_resolver.datamodel.dnssec_schema import DnssecSchema @@ -24,6 +24,8 @@ from knot_resolver.utils.modeling import ConfigSchema from knot_resolver.utils.modeling.base_schema import lazy_default from knot_resolver.utils.modeling.exceptions import AggregateDataValidationError, DataValidationError +WORKERS_MAX = 256 + logger = logging.getLogger(__name__) @@ -42,7 +44,7 @@ def _workers_max_count() -> int: c = _cpu_count() if c: return c * 10 - return WORKERS_MAX_DEFAULT + return WORKERS_MAX def _get_views_tags(views: List[ViewSchema]) -> List[str]: @@ -109,10 +111,10 @@ class KresConfig(ConfigSchema): version: int = 1 nsid: Optional[EscapedStr] = None hostname: Optional[EscapedStr] = None - rundir: WritableDir = lazy_default(WritableDir, str(RUN_DIR_DEFAULT)) + rundir: WritableDir = lazy_default(WritableDir, str(RUN_DIR)) workers: Union[Literal["auto"], IntPositive] = IntPositive(1) - max_workers: IntPositive = IntPositive(WORKERS_MAX_DEFAULT) - management: ManagementSchema = lazy_default(ManagementSchema, {"unix-socket": str(API_SOCK_PATH_DEFAULT)}) + max_workers: IntPositive = IntPositive(WORKERS_MAX) + management: ManagementSchema = lazy_default(ManagementSchema, {"unix-socket": str(API_SOCK_FILE)}) webmgmt: Optional[WebmgmtSchema] = None options: OptionsSchema = OptionsSchema() network: NetworkSchema = NetworkSchema() @@ -178,7 +180,7 @@ class KresConfig(ConfigSchema): workers_max = _workers_max_count() if int(self.workers) > workers_max: raise ValueError( - f"can't run with more workers then the recommended maximum {workers_max} or hardcoded {WORKERS_MAX_DEFAULT}" + f"can't run with more workers then the recommended maximum {workers_max} or hardcoded {WORKERS_MAX}" ) # sanity check @@ -237,7 +239,7 @@ def get_rundir_without_validation(data: Dict[str, Any]) -> WritableDir: Used for initial manager startup. """ - return WritableDir(data["rundir"] if "rundir" in data else RUN_DIR_DEFAULT, object_path="/rundir") + return WritableDir(data["rundir"] if "rundir" in data else str(RUN_DIR), object_path="/rundir") def kres_config_json_schema() -> Dict[str, Any]: diff --git a/python/knot_resolver/datamodel/types/files.py b/python/knot_resolver/datamodel/types/files.py index 5f46c3a1..c2962729 100644 --- a/python/knot_resolver/datamodel/types/files.py +++ b/python/knot_resolver/datamodel/types/files.py @@ -6,7 +6,7 @@ from pathlib import Path from pwd import getpwnam from typing import Any, Dict, Tuple, Type, TypeVar -from knot_resolver.constants import GROUP_DEFAULT, USER_DEFAULT +from knot_resolver.constants import GROUP, USER from knot_resolver.datamodel.globals import get_resolve_root, get_strict_validation from knot_resolver.utils.modeling.base_value_type import BaseValueType @@ -157,8 +157,8 @@ def _kres_accessible(dest_path: Path, perm_mode: _PermissionMode) -> bool: _PermissionMode.EXECUTE: [stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH], } - user_uid = getpwnam(USER_DEFAULT).pw_uid - user_gid = getgrnam(GROUP_DEFAULT).gr_gid + user_uid = getpwnam(USER).pw_uid + user_gid = getgrnam(GROUP).gr_gid dest_stat = os.stat(dest_path) dest_uid = dest_stat.st_uid @@ -195,7 +195,7 @@ class ReadableFile(File): super().__init__(source_value, parents=parents, object_path=object_path) if self.strict_validation and not _kres_accessible(self._value, _PermissionMode.READ): - raise ValueError(f"{USER_DEFAULT}:{GROUP_DEFAULT} has insufficient permissions to read '{self._value}'") + raise ValueError(f"{USER}:{GROUP} has insufficient permissions to read '{self._value}'") class WritableDir(Dir): @@ -213,9 +213,7 @@ class WritableDir(Dir): if self.strict_validation and not _kres_accessible( self._value, _PermissionMode.WRITE | _PermissionMode.EXECUTE ): - raise ValueError( - f"{USER_DEFAULT}:{GROUP_DEFAULT} has insufficient permissions to write/execute '{self._value}'" - ) + raise ValueError(f"{USER}:{GROUP} has insufficient permissions to write/execute '{self._value}'") class WritableFilePath(FilePath): @@ -234,6 +232,4 @@ class WritableFilePath(FilePath): if self.strict_validation and not _kres_accessible( self._value.parent, _PermissionMode.WRITE | _PermissionMode.EXECUTE ): - raise ValueError( - f"{USER_DEFAULT}:{GROUP_DEFAULT} has insufficient permissions to write/execute'{self._value.parent}'" - ) + raise ValueError(f"{USER}:{GROUP} has insufficient permissions to write/execute'{self._value.parent}'") diff --git a/python/knot_resolver/manager/constants.py b/python/knot_resolver/manager/constants.py index 580ac4d8..43b1c04b 100644 --- a/python/knot_resolver/manager/constants.py +++ b/python/knot_resolver/manager/constants.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from typing import TYPE_CHECKING, Optional @@ -7,6 +8,14 @@ if TYPE_CHECKING: from knot_resolver.manager.config_store import ConfigStore +LOGGING_LEVEL_STARTUP = logging.DEBUG +PID_FILE_NAME = "knot-resolver.pid" + +FIX_COUNTER_ATTEMPTS_MAX = 2 +FIX_COUNTER_DECREASE_INTERVAL_SEC = 30 * 60 +WATCHDOG_INTERVAL_SEC: float = 5 + + def kres_cache_dir(config: "KresConfig") -> Path: return config.cache.storage.to_path() diff --git a/python/knot_resolver/manager/logging.py b/python/knot_resolver/manager/logging.py index ff140dcd..c9b44653 100644 --- a/python/knot_resolver/manager/logging.py +++ b/python/knot_resolver/manager/logging.py @@ -4,11 +4,12 @@ import os import sys from typing import Optional -from knot_resolver.constants import LOGGING_LEVEL_STARTUP from knot_resolver.datamodel.config_schema import KresConfig from knot_resolver.datamodel.logging_schema import LogTargetEnum from knot_resolver.manager.config_store import ConfigStore, only_on_real_changes_update +from .constants import LOGGING_LEVEL_STARTUP + logger = logging.getLogger(__name__) diff --git a/python/knot_resolver/manager/main.py b/python/knot_resolver/manager/main.py index 0339790b..dac47bed 100644 --- a/python/knot_resolver/manager/main.py +++ b/python/knot_resolver/manager/main.py @@ -4,12 +4,11 @@ file to allow us to exclude the __main__.py file from black's autoformatting """ import argparse -import os import sys from pathlib import Path from typing import NoReturn -from knot_resolver.constants import CONFIG_FILE_PATH_DEFAULT, CONFIG_FILE_PATH_ENV_VAR, VERSION +from knot_resolver.constants import CONFIG_FILE, VERSION from knot_resolver.manager.logging import logger_startup from knot_resolver.manager.server import start_server from knot_resolver.utils import compat @@ -27,7 +26,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "-c", "--config", - help="Config file to load. Overrides default config location at '" + str(CONFIG_FILE_PATH_ENV_VAR) + "'", + help="Config file to load. Overrides default config location at '" + str(CONFIG_FILE) + "'", type=str, nargs=1, required=False, @@ -44,13 +43,10 @@ def main() -> NoReturn: args = parse_args() # where to look for config - config_env = os.getenv(CONFIG_FILE_PATH_ENV_VAR) if args.config is not None: config_path = Path(args.config[0]) - elif config_env is not None: - config_path = Path(config_env) else: - config_path = CONFIG_FILE_PATH_DEFAULT + config_path = CONFIG_FILE exit_code = compat.asyncio.run(start_server(config=config_path)) sys.exit(exit_code) diff --git a/python/knot_resolver/manager/manager.py b/python/knot_resolver/manager/manager.py index 114ad68e..74d90ec3 100644 --- a/python/knot_resolver/manager/manager.py +++ b/python/knot_resolver/manager/manager.py @@ -6,7 +6,6 @@ from secrets import token_hex from subprocess import SubprocessError from typing import Any, Callable, List, Optional -from knot_resolver.constants import FIX_COUNTER_ATTEMPTS_MAX, FIX_COUNTER_DECREASE_INTERVAL_SEC, WATCHDOG_INTERVAL_SEC from knot_resolver.controller.exceptions import SubprocessControllerException from knot_resolver.controller.interface import Subprocess, SubprocessController, SubprocessStatus, SubprocessType from knot_resolver.controller.registered_workers import command_registered_workers, get_registered_workers_kresids @@ -16,6 +15,8 @@ from knot_resolver.utils.compat.asyncio import create_task from knot_resolver.utils.functional import Result from knot_resolver.utils.modeling.types import NoneType +from .constants import FIX_COUNTER_ATTEMPTS_MAX, FIX_COUNTER_DECREASE_INTERVAL_SEC, WATCHDOG_INTERVAL_SEC + logger = logging.getLogger(__name__) diff --git a/python/knot_resolver/manager/server.py b/python/knot_resolver/manager/server.py index ea2b419c..972b167f 100644 --- a/python/knot_resolver/manager/server.py +++ b/python/knot_resolver/manager/server.py @@ -17,7 +17,7 @@ from aiohttp.web_app import Application from aiohttp.web_response import json_response from aiohttp.web_runner import AppRunner, TCPSite, UnixSite -from knot_resolver.constants import CONFIG_FILE_PATH_DEFAULT, PID_FILE_NAME +from knot_resolver.constants import CONFIG_FILE from knot_resolver.controller import get_best_controller_implementation from knot_resolver.controller.exceptions import SubprocessControllerExecException from knot_resolver.controller.registered_workers import command_single_registered_worker @@ -40,7 +40,7 @@ from knot_resolver.utils.modeling.types import NoneType from knot_resolver.utils.systemd_notify import systemd_notify from .config_store import ConfigStore -from .constants import init_user_constants +from .constants import PID_FILE_NAME, init_user_constants from .exceptions import KresManagerException from .logging import logger_init from .manager import KresManager @@ -506,7 +506,7 @@ async def _sigterm_while_shutting_down(): sys.exit(128 + signal.SIGTERM) -async def start_server(config: Path = CONFIG_FILE_PATH_DEFAULT) -> int: +async def start_server(config: Path = CONFIG_FILE) -> int: # This function is quite long, but it describes how manager runs. So let's silence pylint # pylint: disable=too-many-statements diff --git a/python/knot_resolver/meson.build b/python/knot_resolver/meson.build new file mode 100644 index 00000000..38794daa --- /dev/null +++ b/python/knot_resolver/meson.build @@ -0,0 +1,17 @@ +# python +# SPDX-License-Identifier: GPL-3.0-or-later + +constants_config = configuration_data() +constants_config.set('version', meson.project_version()) +constants_config.set('user', user) +constants_config.set('group', group) +constants_config.set('run_dir', run_dir) +constants_config.set('etc_dir', etc_dir) +constants_config.set('sbin_dir', sbin_dir) +constants_config.set('cache_dir', systemd_cache_dir) + +configure_file( + input: 'constants.py.in', + output: 'constants.py', + configuration: constants_config, +) diff --git a/python/knot_resolver_build_options.py.in b/python/knot_resolver_build_options.py.in deleted file mode 100644 index e6b2accb..00000000 --- a/python/knot_resolver_build_options.py.in +++ /dev/null @@ -1,12 +0,0 @@ -from pathlib import Path - -__version__ = "@kres_version@" - -sbin_dir = Path("@sbin_dir@") -bin_dir = Path("@bin_dir@") -etc_dir = Path("@etc_dir@") -run_dir = Path("@run_dir@") -lib_dir = Path("@lib_dir@") -modules_dir = Path("@modules_dir@") -user = "@user@" -group = "@group@" diff --git a/python/meson.build b/python/meson.build deleted file mode 100644 index 4be61235..00000000 --- a/python/meson.build +++ /dev/null @@ -1,25 +0,0 @@ -# python -# SPDX-License-Identifier: GPL-3.0-or-later - -python_config = configuration_data() -python_config.set('kres_version', meson.project_version()) -python_config.set('sbin_dir', sbin_dir) -python_config.set('bin_dir', bin_dir) -python_config.set('etc_dir', etc_dir) -python_config.set('run_dir', run_dir) -python_config.set('lib_dir', lib_dir) -python_config.set('modules_dir', modules_dir) -python_config.set('user', user) -python_config.set('group', group) - -configure_file( - input: 'knot_resolver_build_options.py.in', - output: 'knot_resolver_build_options.py', - configuration: python_config, -) - -configure_file( - input: 'setup.py.in', - output: 'setup.py', - configuration: python_config, -) diff --git a/python/setup.py.in b/python/setup.py.in deleted file mode 100644 index 57f9a684..00000000 --- a/python/setup.py.in +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import setup - -# TODO: Migrate this to a pyproject.toml once Debian 11 support is dropped. -setup( - name="knot_resolver_build_options", - version="@kres_version@", - description="Knot Resolver helper data for Python", - author="Oto Šťáva", - author_email="oto.stava@nic.cz", - python_requires=">=3.8,<4.0", - py_modules=["knot_resolver_build_options"], -) diff --git a/scripts/poe-tasks/check b/scripts/poe-tasks/check index 9cd529e9..8cae7fdc 100755 --- a/scripts/poe-tasks/check +++ b/scripts/poe-tasks/check @@ -68,6 +68,13 @@ python setup.py --help > /dev/null check_rv $? echo +# check python/knot_resolver/constants.py +echo -e "${yellow}python/knot_resolver/constants.py${reset}" +meson_setup_configure > /dev/null +diff python/knot_resolver/constants.py $build_dir/python/knot_resolver/constants.py +check_rv $? +echo + # check that doc/_static/config.schema.json is the latest echo -e "${yellow}Checking doc/_static/config.schema.json${reset}" python -m knot_resolver.client schema | diff - doc/_static/config.schema.json @@ -82,6 +89,8 @@ else echo -e "${red}These commands might help you:${reset}" echo -e "${red}\tpoe format${reset}" echo -e "${red}\tpoe gen-setuppy${reset}" + echo -e "${red}\tpoe gen-constantspy${reset}" + echo -e "${red}\tpoe doc-schema${reset}" echo -e "${red}That's not great. Could you please fix that?${reset} 😲😟" fi diff --git a/scripts/poe-tasks/clean b/scripts/poe-tasks/clean index ff69ad69..c8f596ca 100755 --- a/scripts/poe-tasks/clean +++ b/scripts/poe-tasks/clean @@ -5,16 +5,18 @@ src_dir="$(dirname "$(realpath "$0")")" source $src_dir/utils/_env.sh echo +echo -------------------------------------------- echo Removing mypy, pytest and other cached files echo -------------------------------------------- rm -vrf .coverage .mypy_cache .pytest_cache - echo -echo Removing meson build directories and files echo ------------------------------------------ -rm -vrf .build_kresd .install_kresd build build_doc dist - +echo Removing Meson build directories and files +echo ------------------------------------------ +rm -vrf "$build_dir" "$build_dev_dir" "$KRES_DEV_INSTALL_DIR" build dist echo +echo ------------------------------------------ echo Removing __pycache__ directories and files echo ------------------------------------------ find . -type d -name '__pycache__' -prune -exec rm -vrf {} + +echo
\ No newline at end of file diff --git a/scripts/poe-tasks/configure b/scripts/poe-tasks/configure index ea2bdeb3..dc4aa074 100755 --- a/scripts/poe-tasks/configure +++ b/scripts/poe-tasks/configure @@ -4,10 +4,11 @@ src_dir="$(dirname "$(realpath "$0")")" source $src_dir/utils/_env.sh -reconfigure='' -if [ -f .build_kresd/ninja.build ]; then - reconfigure='--reconfigure' -fi -meson setup .build_kresd "$reconfigure" --prefix=$(realpath .install_kresd) -Duser=$USER -Dgroup=$(id -gn) "$@" - -build_kresd +echo +meson_setup_configure_dev +echo +echo ----------------------------------------------- +echo Copying constants.py module configured by Meson +echo ----------------------------------------------- +cp -v $build_dev_dir/python/knot_resolver/constants.py $gitroot/python/knot_resolver/constants.py +echo diff --git a/scripts/poe-tasks/doc b/scripts/poe-tasks/doc new file mode 100755 index 00000000..ed01f426 --- /dev/null +++ b/scripts/poe-tasks/doc @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# ensure consistent behaviour +src_dir="$(dirname "$(realpath "$0")")" +source $src_dir/utils/_env.sh + +meson_setup_configure -Ddoc=enabled + +echo ---------------------------------- +echo Building documentation using ninja +echo ---------------------------------- +ninja -C $build_dir doc +echo diff --git a/scripts/poe-tasks/doc-schema b/scripts/poe-tasks/doc-schema new file mode 100755 index 00000000..33ea3bd6 --- /dev/null +++ b/scripts/poe-tasks/doc-schema @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# ensure consistent behaviour +src_dir="$(dirname "$(realpath "$0")")" +source $src_dir/utils/_env.sh + +schema_file="$gitroot/doc/_static/config.schema.json" + +meson_setup_configure > /dev/null +cp $build_dir/python/knot_resolver/constants.py $gitroot/python/knot_resolver/constants.py +python -m knot_resolver.client schema > $schema_file + +echo New configuration JSON schem saved to $schema_file
\ No newline at end of file diff --git a/scripts/poe-tasks/docs b/scripts/poe-tasks/docs deleted file mode 100755 index 0364cf72..00000000 --- a/scripts/poe-tasks/docs +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -# ensure consistent behaviour -src_dir="$(dirname "$(realpath "$0")")" -source $src_dir/utils/_env.sh - -echo Building Knot Resolver documentation -echo ------------------------------------ -meson build_doc -Ddoc=enabled -ninja -C build_doc doc diff --git a/scripts/poe-tasks/examples b/scripts/poe-tasks/examples index 8b0be2c4..d4437203 100755 --- a/scripts/poe-tasks/examples +++ b/scripts/poe-tasks/examples @@ -7,5 +7,5 @@ source $src_dir/utils/_env.sh # validate all configuration examples for example in $PWD/etc/config/config.example.*.yaml; do - poe kresctl validate --no-strict $example; + python3 -m knot_resolver.client validate --no-strict $example; done diff --git a/scripts/poe-tasks/gen-constantspy b/scripts/poe-tasks/gen-constantspy new file mode 100755 index 00000000..0f3bb3b6 --- /dev/null +++ b/scripts/poe-tasks/gen-constantspy @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# ensure consistent behaviour +src_dir="$(dirname "$(realpath "$0")")" +source $src_dir/utils/_env.sh + +echo +meson_setup_configure +echo +echo ----------------------------------------------- +echo Copying constants.py module configured by Meson +echo ----------------------------------------------- +cp -v $build_dir/python/knot_resolver/constants.py $gitroot/python/knot_resolver/constants.py +echo
\ No newline at end of file diff --git a/scripts/poe-tasks/kresctl b/scripts/poe-tasks/kresctl index 50e028fc..87a96e30 100755 --- a/scripts/poe-tasks/kresctl +++ b/scripts/poe-tasks/kresctl @@ -4,5 +4,9 @@ src_dir="$(dirname "$(realpath "$0")")" source $src_dir/utils/_env.sh +echo +is_build_dev_dir_configured +echo + # run knot-resolver client -python3 -m knot_resolver.client $@ +python3 -m knot_resolver.client --config "$KRES_DEV_CONFIG_FILE" "$@" diff --git a/scripts/poe-tasks/man b/scripts/poe-tasks/man deleted file mode 100755 index 3cdafb34..00000000 --- a/scripts/poe-tasks/man +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -# ensure consistent behaviour -src_dir="$(dirname "$(realpath "$0")")" -source $src_dir/utils/_env.sh - -build_kresd - -man -l .install_kresd/share/man/man8/$1* diff --git a/scripts/poe-tasks/run b/scripts/poe-tasks/run index 90f02632..4d3ebc8c 100755 --- a/scripts/poe-tasks/run +++ b/scripts/poe-tasks/run @@ -4,11 +4,18 @@ src_dir="$(dirname "$(realpath "$0")")" source $src_dir/utils/_env.sh -build_kresd +echo +ninja_dev_install +echo +echo ------------------------------------ +echo Creating missing runtime directories +echo ------------------------------------ +mkdir -vp $KRES_DEV_INSTALL_DIR/run/knot-resolver $KRES_DEV_INSTALL_DIR/var/cache/knot-resolver echo -echo Building Knot Resolver Manager native extensions -echo ------------------------------------------------ +echo --------------------------------------- +echo Building Python komponents using Poetry +echo --------------------------------------- poetry build # copy native modules from build directory to source directory shopt -s globstar @@ -22,8 +29,7 @@ shopt -u globstar shopt -u nullglob echo -echo The Knot Resolver management API is accessible on http://localhost:5000 -echo ----------------------------------------------------------------------- - -# run the knot-resolver manager -python3 -m knot_resolver.manager $@ +echo -------------------------------------- +echo Starting Knot Resolver wit the Manager +echo -------------------------------------- +python3 -m knot_resolver.manager --config "$KRES_DEV_CONFIG_FILE" "$@" diff --git a/scripts/poe-tasks/utils/_env.sh b/scripts/poe-tasks/utils/_env.sh index 1ea73a39..66cece83 100644 --- a/scripts/poe-tasks/utils/_env.sh +++ b/scripts/poe-tasks/utils/_env.sh @@ -17,6 +17,11 @@ if test -z "$gitroot"; then fi cd $gitroot +# build dirs +build_dir="$gitroot/.build" +build_dev_dir="$gitroot/.build_dev" +install_dev_dir="$gitroot/.install_dev" + # ensure consistent environment with virtualenv if test -z "$VIRTUAL_ENV" -a "$CI" != "true" -a -z "$KNOT_ENV"; then echo -e "${yellow}You are NOT running the script within the project's virtual environment.${reset}" @@ -34,39 +39,66 @@ PATH="$PATH:$gitroot/node_modules/.bin" # fail even on unbound variables set -o nounset -# create runtime directories -if [ -z "${KRES_CONFIG_DIR:-}" ]; then - KRES_CONFIG_DIR="$gitroot/etc/config" +# Set enviromental variables if not +if [ -z "${KRES_DEV_INSTALL_DIR:-}" ]; then + KRES_DEV_INSTALL_DIR="$install_dev_dir" fi -mkdir -p "$KRES_CONFIG_DIR/runtime" "$KRES_CONFIG_DIR/cache" - -# env variables -if [ -z "${KRES_MANAGER_CONFIG:-}" ]; then - KRES_MANAGER_CONFIG="$KRES_CONFIG_DIR/config.dev.yaml" +if [ -z "${KRES_DEV_CONFIG_FILE:-}" ]; then + KRES_DEV_CONFIG_FILE="$gitroot/etc/config/config.dev.yaml" fi +export KRES_DEV_INSTALL_DIR +export KRES_DEV_CONFIG_FILE -if [ -z "${KRES_MANAGER_API_SOCK:-}" ]; then - KRES_MANAGER_API_SOCK="$KRES_CONFIG_DIR/manager.sock" -fi -export KRES_MANAGER_CONFIG -export KRES_MANAGER_API_SOCK +function meson_setup_configure { + local reconfigure='' + if [ -d $build_dir ]; then + reconfigure='--reconfigure' + fi + echo --------------------------------------- + echo Configuring build directory using Meson + echo --------------------------------------- + meson setup \ + $build_dir \ + $reconfigure \ + --prefix=/usr \ + "$@" +} -function build_kresd { - if [ -d .build_kresd ]; then - echo - echo Building Knot Resolver - echo ---------------------- - echo -e "${blue}In case of an compilation error, run this command to try to fix it:${reset}" - echo -e "\t${blue}rm -r $(realpath .install_kresd) $(realpath .build_kresd)${reset}" - echo - ninja -C .build_kresd - ninja install -C .build_kresd - export PYTHONPATH="$(realpath .build_kresd/python):${PYTHONPATH:-}" - else +function meson_setup_configure_dev { + local reconfigure='' + if [ -d $build_dev_dir ]; then + reconfigure='--reconfigure' + fi + echo --------------------------------------- + echo Configuring build directory using Meson + echo --------------------------------------- + meson setup \ + $build_dev_dir \ + $reconfigure \ + --prefix=$KRES_DEV_INSTALL_DIR \ + -D user=$(id -un) \ + -D group=$(id -gn) \ + "$@" +} + +function is_build_dev_dir_configured { + if [ ! -d $build_dev_dir ]; then echo - echo Knot Resolver daemon is not configured. - echo "Please run './poe configure' (optionally with additional Meson arguments)" + echo Knot Resolver build directory is not configured by Meson. + echo "Please run './poe configure' (optionally with additional Meson arguments)". echo exit 2 fi } + +function ninja_dev_install { + + is_build_dev_dir_configured + + echo + echo -------------------------------------------- + echo Building/installing C komponents using Ninja + echo -------------------------------------------- + ninja -C $build_dev_dir + ninja install -C $build_dev_dir +} diff --git a/tests/packaging/systemd_service.sh b/tests/packaging/systemd_service.sh index 9dec16ab..a4425385 100755 --- a/tests/packaging/systemd_service.sh +++ b/tests/packaging/systemd_service.sh @@ -9,6 +9,14 @@ if test "$(id -u)" -ne 0; then exit 1 fi +# SKIP test when systemd isn't PID 1 +if [[ -d /run/systemd/system ]] ; then + echo "systemd detected -> RUN systemd tests" +else + echo "systemd not detected -> SKIP systemd tests" + exit 77 +fi + # We will be starting a systemd service, but another tests might do the same # so this makes sure there is nothing left after we exit trap "systemctl stop knot-resolver.service" EXIT |