From 735a00592ca378ee4b6fdee823893f7af9d2945c Mon Sep 17 00:00:00 2001 From: Oto Šťáva Date: Thu, 21 Mar 2024 11:49:05 +0100 Subject: modules/stats: split stats.list() into sub-objects --- modules/stats/stats.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'modules') diff --git a/modules/stats/stats.c b/modules/stats/stats.c index ca3a932c..968b46cb 100644 --- a/modules/stats/stats.c +++ b/modules/stats/stats.c @@ -57,10 +57,18 @@ enum const_metric { }; struct const_metric_elm { const char *key; + const char *sup_key; + const char *sub_key; size_t val; }; static struct const_metric_elm const_metrics[] = { - #define X(a,b) [metric_ ## a ## _ ## b] = { #a "." #b, 0 }, + #define X(a,b) \ + [metric_ ## a ## _ ## b] = { \ + .key = #a "." #b, \ + .sup_key = #a, \ + .sub_key = #b, \ + .val = 0 \ + }, CONST_METRICS(X) #undef X }; @@ -376,8 +384,15 @@ static char* stats_list(void *env, struct kr_module *module, const char *args) size_t args_len = args ? strlen(args) : 0; for (unsigned i = 0; i < metric_const_end; ++i) { struct const_metric_elm *elm = &const_metrics[i]; - if (!args || strncmp(elm->key, args, args_len) == 0) { - json_append_member(root, elm->key, json_mknumber(elm->val)); + if (!args || strcmp(elm->sup_key, args) == 0) { + JsonNode *sup = json_find_member(root, elm->sup_key); + if (!sup) { + sup = json_mkobject(); + json_append_member(root, elm->sup_key, sup); + } + if (kr_fails_assert(sup)) + break; + json_append_member(sup, elm->sub_key, json_mknumber(elm->val)); } } struct list_entry_context ctx = { -- cgit v1.2.3 From 5a212e337a56e5b365487cf4d25f907b5c6e42c1 Mon Sep 17 00:00:00 2001 From: Oto Šťáva Date: Thu, 21 Mar 2024 12:17:59 +0100 Subject: modules/{stats,http}: fix built-in Prometheus and tests --- modules/http/prometheus.lua | 8 ++++-- modules/stats/test.integr/kresd_config.j2 | 48 +++++++++++++++++-------------- 2 files changed, 32 insertions(+), 24 deletions(-) (limited to 'modules') diff --git a/modules/http/prometheus.lua b/modules/http/prometheus.lua index 3218552f..3164e9d5 100644 --- a/modules/http/prometheus.lua +++ b/modules/http/prometheus.lua @@ -15,8 +15,12 @@ local function merge(t, results, prefix) for _, result in pairs(results) do if type(result) == 'table' then for k, v in pairs(result) do - local val = t[prefix..k] - t[prefix..k] = (val or 0) + v + if type(result) == 'table' then + merge(t, result, prefix..k..'.') + else + local val = t[prefix..k] + t[prefix..k] = (val or 0) + v + end end end end diff --git a/modules/stats/test.integr/kresd_config.j2 b/modules/stats/test.integr/kresd_config.j2 index 872ce2e3..d0707691 100644 --- a/modules/stats/test.integr/kresd_config.j2 +++ b/modules/stats/test.integr/kresd_config.j2 @@ -9,32 +9,36 @@ FWD_TARGET = policy.FORWARD('192.0.2.1') function check_stats(got) log_info(ffi.C.LOG_GRP_TESTS, 'checking if stat values match expected values:') local expected = { - ['answer.cd'] = 2, - ['answer.cached'] = 1, - ['answer.nodata'] = 1, - ['answer.noerror'] = 2, - ['answer.nxdomain'] = 1, - ['answer.servfail'] = 2, - ['answer.edns0'] = 6, - ['answer.ra'] = 6, - ['answer.rd'] = 5, - ['answer.do'] = 1, - ['answer.ad'] = 0, - ['answer.tc'] = 0, - ['answer.aa'] = 0, - ['answer.total'] = 6 + ['answer'] = { + ['cd'] = 2, + ['cached'] = 1, + ['nodata'] = 1, + ['noerror'] = 2, + ['nxdomain'] = 1, + ['servfail'] = 2, + ['edns0'] = 6, + ['ra'] = 6, + ['rd'] = 5, + ['do'] = 1, + ['ad'] = 0, + ['tc'] = 0, + ['aa'] = 0, + ['total'] = 6 + } } print(table_print(expected)) local ok = true - for key, expval in pairs(expected) do - if got[key] ~= expval then - log_info(ffi.C.LOG_GRP_TESTS, - 'ERROR: stats key ' .. key - .. ' has unexpected value' - .. ' (expected ' .. tostring(expval) - .. ' got ' .. tostring(got[key] .. ')')) - ok = false + for sup_key, sup in pairs(expected) do + for sub_key, expval in pairs(sup) do + if got[sup_key][sub_key] ~= expval then + log_info(ffi.C.LOG_GRP_TESTS, + 'ERROR: stats key ' .. key + .. ' has unexpected value' + .. ' (expected ' .. tostring(expval) + .. ' got ' .. tostring(got[key] .. ')')) + ok = false + end end end if ok then -- cgit v1.2.3 From 5bc1e0370b47fd9e5a011a35f2a1ba0329f95457 Mon Sep 17 00:00:00 2001 From: Aleš Mrázek Date: Wed, 17 Apr 2024 15:41:52 +0200 Subject: modules: predict: prefetching expired records has been removed --- modules/predict/README.rst | 23 ++++------------------- modules/predict/predict.lua | 14 -------------- 2 files changed, 4 insertions(+), 33 deletions(-) (limited to 'modules') diff --git a/modules/predict/README.rst b/modules/predict/README.rst index 966c4ca4..0bc85671 100644 --- a/modules/predict/README.rst +++ b/modules/predict/README.rst @@ -2,34 +2,19 @@ .. _mod-predict: -Prefetching records -=================== - -The ``predict`` module helps to keep the cache hot by prefetching records. -It can utilize two independent mechanisms to select the records which should be refreshed: -expiring records and prediction. - -Expiring records ----------------- - -This mechanism is always active when the predict module is loaded and it is not configurable. - -Any time the resolver answers with records that are about to expire, -they get refreshed. (see :c:func:`is_expiring`) -That improves latency for records which get frequently queried, relatively to their TTL. - Prediction ---------- -The predict module can also learn usage patterns and repetitive queries, +``predict`` is an experimental module that tries to help keep the cache hot by prefetching records using a prediction mechanism to select records which should be refreshed. + +The module can learn usage patterns and repetitive queries, though this mechanism is a prototype and **not recommended** for use in production or with high traffic. For example, if it makes a query every day at 18:00, the resolver expects that it is needed by that time and prefetches it ahead of time. This is helpful to minimize the perceived latency and keeps the cache hot. -You can disable prediction by configuring ``period = 0``. -Otherwise it will load the required :ref:`stats ` module if not present, +It will load the required :ref:`stats ` module if not present, and it will use its :func:`stats.frequent` table and clear it periodically. .. tip:: The tracking window and period length determine memory requirements. If you have a server with relatively fast query turnover, keep the period low (hour for start) and shorter tracking window (5 minutes). For personal slower resolver, keep the tracking window longer (i.e. 30 minutes) and period longer (a day), as the habitual queries occur daily. Experiment to get the best results. diff --git a/modules/predict/predict.lua b/modules/predict/predict.lua index 0117fd52..38fb7258 100644 --- a/modules/predict/predict.lua +++ b/modules/predict/predict.lua @@ -172,18 +172,4 @@ function predict.config(config) predict.init() end -predict.layer = { - -- Prefetch all expiring (sub-)queries immediately after the request finishes. - -- Doing that immediately is simplest and avoids creating (new) large bursts of activity. - finish = function (_, req) - local qrys = req.rplan.resolved - for i = 0, (tonumber(qrys.len) - 1) do -- size_t doesn't work for some reason - local qry = qrys.at[i] - if qry.flags.EXPIRING == true then - resolve(kres.dname2str(qry.sname), qry.stype, qry.sclass, {'NO_CACHE'}) - end - end - end -} - return predict -- cgit v1.2.3 From ee018de76f07d0e0da433913966708d01476aa48 Mon Sep 17 00:00:00 2001 From: Aleš Mrázek Date: Wed, 17 Apr 2024 15:55:02 +0200 Subject: modules: prefetch: new module for prefetching expiring records --- distro/pkg/deb/knot-resolver-core.install | 1 + distro/pkg/rpm/knot-resolver.spec | 1 + modules/meson.build | 1 + modules/prefetch/README.rst | 18 ++++++++++++++++++ modules/prefetch/prefetch.lua | 21 +++++++++++++++++++++ 5 files changed, 42 insertions(+) create mode 100644 modules/prefetch/README.rst create mode 100644 modules/prefetch/prefetch.lua (limited to 'modules') diff --git a/distro/pkg/deb/knot-resolver-core.install b/distro/pkg/deb/knot-resolver-core.install index 74ff1c57..1e57ac9b 100644 --- a/distro/pkg/deb/knot-resolver-core.install +++ b/distro/pkg/deb/knot-resolver-core.install @@ -18,6 +18,7 @@ usr/lib/knot-resolver/kres_modules/experimental_dot_auth.lua usr/lib/knot-resolver/kres_modules/graphite.lua usr/lib/knot-resolver/kres_modules/policy.lua usr/lib/knot-resolver/kres_modules/predict.lua +usr/lib/knot-resolver/kres_modules/prefetch.lua usr/lib/knot-resolver/kres_modules/prefill.lua usr/lib/knot-resolver/kres_modules/priming.lua usr/lib/knot-resolver/kres_modules/rebinding.lua diff --git a/distro/pkg/rpm/knot-resolver.spec b/distro/pkg/rpm/knot-resolver.spec index acf313f8..5fd08f8d 100644 --- a/distro/pkg/rpm/knot-resolver.spec +++ b/distro/pkg/rpm/knot-resolver.spec @@ -299,6 +299,7 @@ getent passwd knot-resolver >/dev/null || useradd -r -g knot-resolver -d %{_sysc %{_libdir}/knot-resolver/kres_modules/graphite.lua %{_libdir}/knot-resolver/kres_modules/policy.lua %{_libdir}/knot-resolver/kres_modules/predict.lua +%{_libdir}/knot-resolver/kres_modules/prefetch.lua %{_libdir}/knot-resolver/kres_modules/prefill.lua %{_libdir}/knot-resolver/kres_modules/priming.lua %{_libdir}/knot-resolver/kres_modules/rebinding.lua diff --git a/modules/meson.build b/modules/meson.build index 38612254..73444e99 100644 --- a/modules/meson.build +++ b/modules/meson.build @@ -7,6 +7,7 @@ lua_mod_src = [ # add lua modules without separate meson.build files('dns64/dns64.lua'), files('etcd/etcd.lua'), files('graphite/graphite.lua'), + files('prefetch/prefetch.lua'), files('predict/predict.lua'), files('prefill/prefill.lua'), files('priming/priming.lua'), diff --git a/modules/prefetch/README.rst b/modules/prefetch/README.rst new file mode 100644 index 00000000..4d5a5e3e --- /dev/null +++ b/modules/prefetch/README.rst @@ -0,0 +1,18 @@ +.. SPDX-License-Identifier: GPL-3.0-or-later + +.. _mod-prefetch: + +Expiring records +---------------- + +The ``prefetch`` module helps to keep the cache hot by prefetching expiring records. + +This mechanism is activated when the module is loaded and it is not configurable. + +.. code-block:: lua + + modules.load('prefetch') + + +Any time the resolver answers with records that are about to expire, they get refreshed. (see :c:func:`is_expiring`) +That improves latency for records which get frequently queried, relatively to their TTL. diff --git a/modules/prefetch/prefetch.lua b/modules/prefetch/prefetch.lua new file mode 100644 index 00000000..673cf3e5 --- /dev/null +++ b/modules/prefetch/prefetch.lua @@ -0,0 +1,21 @@ +-- SPDX-License-Identifier: GPL-3.0-or-later +-- Speculative prefetching for repetitive and soon-expiring records to reduce latency. +-- @module prefetch +local prefetch = {} + + +prefetch.layer = { + -- Prefetch all expiring (sub-)queries immediately after the request finishes. + -- Doing that immediately is simplest and avoids creating (new) large bursts of activity. + finish = function (_, req) + local qrys = req.rplan.resolved + for i = 0, (tonumber(qrys.len) - 1) do -- size_t doesn't work for some reason + local qry = qrys.at[i] + if qry.flags.EXPIRING == true then + resolve(kres.dname2str(qry.sname), qry.stype, qry.sclass, {'NO_CACHE'}) + end + end + end +} + +return prefetch -- cgit v1.2.3 From 50de6fb35eda7cb6612b75b3938e91fbe6099a9c Mon Sep 17 00:00:00 2001 From: Oto Šťáva Date: Fri, 19 Apr 2024 16:18:51 +0200 Subject: modules/stats: make custom stats hierarchical Forgotten feature from !1527 --- modules/stats/stats.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) (limited to 'modules') diff --git a/modules/stats/stats.c b/modules/stats/stats.c index 968b46cb..a8a29de2 100644 --- a/modules/stats/stats.c +++ b/modules/stats/stats.c @@ -367,8 +367,31 @@ static int list_entry(const char *key, uint32_t key_len, trie_val_t *val, void * if (!key_matches_prefix(key, key_len, ctx->key_prefix, ctx->key_prefix_len)) return 0; size_t number = (size_t) *val; - auto_free char *key_nt = strndup(key, key_len); - json_append_member(ctx->root, key_nt, json_mknumber(number)); + + uint32_t dot_index = 0; + for (uint32_t i = 0; i < key_len; i++) { + if (!key[i]) + break; + if (key[i] == '.') { + dot_index = i; + } + } + + if (dot_index) { + auto_free char *sup_key_nt = strndup(key, dot_index); + auto_free char *sub_key_nt = strndup(key + dot_index + 1, key_len - dot_index - 1); + JsonNode *sup = json_find_member(ctx->root, sup_key_nt); + if (!sup) { + sup = json_mkobject(); + json_append_member(ctx->root, sup_key_nt, sup); + } + if (kr_fails_assert(sup)) + return 0; + json_append_member(sup, sub_key_nt, json_mknumber(number)); + } else { + auto_free char *key_nt = strndup(key, key_len); + json_append_member(ctx->root, key_nt, json_mknumber(number)); + } return 0; } -- cgit v1.2.3 From e32df41e6f67ea8b02e0b5deb613e0a74f8205b2 Mon Sep 17 00:00:00 2001 From: Oto Šťáva Date: Thu, 2 May 2024 12:21:00 +0200 Subject: modules/*/meson.build: add missing dependencies --- .github/workflows/macOS.yaml | 4 +++- modules/bogus_log/meson.build | 2 +- modules/dnstap/meson.build | 3 +-- modules/edns_keepalive/meson.build | 2 +- modules/extended_error/meson.build | 4 +--- modules/hints/meson.build | 5 +---- modules/http/meson.build | 2 +- modules/meson.build | 7 +++++++ modules/nsid/meson.build | 5 +---- modules/refuse_nord/meson.build | 2 +- modules/stats/meson.build | 4 +--- tests/unit/meson.build | 7 ++++--- utils/cache_gc/meson.build | 2 ++ 13 files changed, 25 insertions(+), 24 deletions(-) (limited to 'modules') diff --git a/.github/workflows/macOS.yaml b/.github/workflows/macOS.yaml index fd834883..f7fe0907 100644 --- a/.github/workflows/macOS.yaml +++ b/.github/workflows/macOS.yaml @@ -47,4 +47,6 @@ jobs: env: MALLOC_CHECK_: 3 MALLOC_PERTURB_: 223 - run: echo "quit()" | ${HOME}/.local/usr/sbin/kresd -a 127.0.0.1@53535 . + run: | + export DYLD_FALLBACK_LIBRARY_PATH="${DYLD_FALLBACK_LIBRARY_PATH}:${HOME}/.local/usr/lib/" + echo "quit()" | ${HOME}/.local/usr/sbin/kresd -a 127.0.0.1@53535 . diff --git a/modules/bogus_log/meson.build b/modules/bogus_log/meson.build index e2faed58..3fa8d3cf 100644 --- a/modules/bogus_log/meson.build +++ b/modules/bogus_log/meson.build @@ -9,7 +9,7 @@ c_src_lint += bogus_log_src bogus_log_mod = shared_module( 'bogus_log', bogus_log_src, - dependencies: libknot, + dependencies: mod_deps, include_directories: mod_inc_dir, name_prefix: '', install: true, diff --git a/modules/dnstap/meson.build b/modules/dnstap/meson.build index e8a94bf1..038bf3e3 100644 --- a/modules/dnstap/meson.build +++ b/modules/dnstap/meson.build @@ -43,11 +43,10 @@ if build_dnstap dnstap_mod = shared_module( 'dnstap', dnstap_src, - dependencies: [ + dependencies: mod_deps + [ declare_dependency(sources: dnstap_pb), libfstrm, libprotobuf_c, - libknot, ], include_directories: mod_inc_dir, name_prefix: '', diff --git a/modules/edns_keepalive/meson.build b/modules/edns_keepalive/meson.build index d125ec45..8370cdb1 100644 --- a/modules/edns_keepalive/meson.build +++ b/modules/edns_keepalive/meson.build @@ -9,7 +9,7 @@ c_src_lint += edns_keepalive_src edns_keepalive_mod = shared_module( 'edns_keepalive', edns_keepalive_src, - dependencies: libknot, + dependencies: mod_deps, include_directories: mod_inc_dir, name_prefix: '', install: true, diff --git a/modules/extended_error/meson.build b/modules/extended_error/meson.build index 15a1772f..9de514af 100644 --- a/modules/extended_error/meson.build +++ b/modules/extended_error/meson.build @@ -9,9 +9,7 @@ c_src_lint += extended_error_src extended_error_mod = shared_module( 'extended_error', extended_error_src, - dependencies: [ - libknot, - ], + dependencies: mod_deps, include_directories: mod_inc_dir, name_prefix: '', install: true, diff --git a/modules/hints/meson.build b/modules/hints/meson.build index b8379181..d5046cb4 100644 --- a/modules/hints/meson.build +++ b/modules/hints/meson.build @@ -9,10 +9,7 @@ c_src_lint += hints_src hints_mod = shared_module( 'hints', hints_src, - dependencies: [ - libknot, - luajit, - ], + dependencies: mod_deps, include_directories: mod_inc_dir, name_prefix: '', install: true, diff --git a/modules/http/meson.build b/modules/http/meson.build index a36e9ebd..9d20c929 100644 --- a/modules/http/meson.build +++ b/modules/http/meson.build @@ -45,7 +45,7 @@ install_subdir( ) # auxiliary debug library for HTTP module - doesn't compile on Cygwin -if openssl.found() and host_machine.system() != 'cygwin' +if openssl.found() and host_machine.system() not in [ 'cygwin', 'darwin' ] debug_opensslkeylog_mod = shared_module( 'debug_opensslkeylog', ['debug_opensslkeylog.c'], diff --git a/modules/meson.build b/modules/meson.build index 38612254..48bd4781 100644 --- a/modules/meson.build +++ b/modules/meson.build @@ -35,6 +35,13 @@ integr_tests += [ mod_inc_dir = include_directories('..', '../contrib', luajit.get_pkgconfig_variable('includedir')) +mod_deps = [ + contrib_dep, + libknot, + libuv, + luajit, +] + # handle more complex C/LUA modules separately subdir('bogus_log') # cookies module is not currently used diff --git a/modules/nsid/meson.build b/modules/nsid/meson.build index 354e70b9..3c418bc9 100644 --- a/modules/nsid/meson.build +++ b/modules/nsid/meson.build @@ -9,10 +9,7 @@ c_src_lint += nsid_src nsid_mod = shared_module( 'nsid', nsid_src, - dependencies: [ - libknot, - luajit, - ], + dependencies: mod_deps, include_directories: mod_inc_dir, name_prefix: '', install: true, diff --git a/modules/refuse_nord/meson.build b/modules/refuse_nord/meson.build index 5142ded6..7dc8b888 100644 --- a/modules/refuse_nord/meson.build +++ b/modules/refuse_nord/meson.build @@ -13,7 +13,7 @@ c_src_lint += refuse_nord_src refuse_nord_mod = shared_module( 'refuse_nord', refuse_nord_src, - dependencies: libknot, + dependencies: mod_deps, include_directories: mod_inc_dir, name_prefix: '', install: true, diff --git a/modules/stats/meson.build b/modules/stats/meson.build index cb4ccd68..e1f4a49c 100644 --- a/modules/stats/meson.build +++ b/modules/stats/meson.build @@ -14,9 +14,7 @@ integr_tests += [ stats_mod = shared_module( 'stats', stats_src, - dependencies: [ - libknot, - ], + dependencies: mod_deps, include_directories: mod_inc_dir, name_prefix: '', install: true, diff --git a/tests/unit/meson.build b/tests/unit/meson.build index b10789cc..747f1d3b 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -10,7 +10,7 @@ mock_cmodule_mod = shared_module( 'mock_cmodule', mock_cmodule_src, name_prefix: '', - dependencies: libknot, + dependencies: mod_deps, include_directories: mod_inc_dir, ) @@ -20,10 +20,11 @@ foreach unit_test : unit_tests unit_test[0], unit_test[1], dependencies: [ + cmocka, contrib_dep, - libkres_dep, libknot, - cmocka, + libkres_dep, + libuv, lmdb, ], ) diff --git a/utils/cache_gc/meson.build b/utils/cache_gc/meson.build index 4c82b8da..6ed86afe 100644 --- a/utils/cache_gc/meson.build +++ b/utils/cache_gc/meson.build @@ -18,6 +18,8 @@ if build_utils contrib_dep, libkres_dep, libknot, + libuv, + lmdb, ], install: true, install_dir: get_option('sbindir'), -- cgit v1.2.3