Merge remote-tracking branch 'origin/6.0' into rrl-wip

17 files changed, 135 insertions, 82 deletions

diff --git a/modules/bogus_log/meson.build b/modules/bogus_log/meson.build
index e2faed58..3fa8d3cf 100644
--- a/modules/bogus_log/meson.build
+++ b/modules/bogus_log/meson.build
@@ -9,7 +9,7 @@ c_src_lint += bogus_log_src
 bogus_log_mod = shared_module(
   'bogus_log',
   bogus_log_src,
-  dependencies: libknot,
+  dependencies: mod_deps,
   include_directories: mod_inc_dir,
   name_prefix: '',
   install: true,
diff --git a/modules/dnstap/meson.build b/modules/dnstap/meson.build
index e8a94bf1..038bf3e3 100644
--- a/modules/dnstap/meson.build
+++ b/modules/dnstap/meson.build
@@ -43,11 +43,10 @@ if build_dnstap
   dnstap_mod = shared_module(
     'dnstap',
     dnstap_src,
-    dependencies: [
+    dependencies: mod_deps + [
       declare_dependency(sources: dnstap_pb),
       libfstrm,
       libprotobuf_c,
-      libknot,
     ],
     include_directories: mod_inc_dir,
     name_prefix: '',
diff --git a/modules/edns_keepalive/meson.build b/modules/edns_keepalive/meson.build
index d125ec45..8370cdb1 100644
--- a/modules/edns_keepalive/meson.build
+++ b/modules/edns_keepalive/meson.build
@@ -9,7 +9,7 @@ c_src_lint += edns_keepalive_src
 edns_keepalive_mod = shared_module(
   'edns_keepalive',
   edns_keepalive_src,
-  dependencies: libknot,
+  dependencies: mod_deps,
   include_directories: mod_inc_dir,
   name_prefix: '',
   install: true,
diff --git a/modules/extended_error/meson.build b/modules/extended_error/meson.build
index 15a1772f..9de514af 100644
--- a/modules/extended_error/meson.build
+++ b/modules/extended_error/meson.build
@@ -9,9 +9,7 @@ c_src_lint += extended_error_src
 extended_error_mod = shared_module(
   'extended_error',
   extended_error_src,
-  dependencies: [
-    libknot,
-  ],
+  dependencies: mod_deps,
   include_directories: mod_inc_dir,
   name_prefix: '',
   install: true,
diff --git a/modules/hints/meson.build b/modules/hints/meson.build
index b8379181..d5046cb4 100644
--- a/modules/hints/meson.build
+++ b/modules/hints/meson.build
@@ -9,10 +9,7 @@ c_src_lint += hints_src
 hints_mod = shared_module(
   'hints',
   hints_src,
-  dependencies: [
-    libknot,
-    luajit,
-  ],
+  dependencies: mod_deps,
   include_directories: mod_inc_dir,
   name_prefix: '',
   install: true,
diff --git a/modules/http/meson.build b/modules/http/meson.build
index a36e9ebd..9d20c929 100644
--- a/modules/http/meson.build
+++ b/modules/http/meson.build
@@ -45,7 +45,7 @@ install_subdir(
 )
 
 # auxiliary debug library for HTTP module - doesn't compile on Cygwin
-if openssl.found() and host_machine.system() != 'cygwin'
+if openssl.found() and host_machine.system() not in [ 'cygwin', 'darwin' ]
   debug_opensslkeylog_mod = shared_module(
     'debug_opensslkeylog',
     ['debug_opensslkeylog.c'],
diff --git a/modules/http/prometheus.lua b/modules/http/prometheus.lua
index 3218552f..3164e9d5 100644
--- a/modules/http/prometheus.lua
+++ b/modules/http/prometheus.lua
@@ -15,8 +15,12 @@ local function merge(t, results, prefix)
 	for _, result in pairs(results) do
 		if type(result) == 'table' then
 			for k, v in pairs(result) do
-				local val = t[prefix..k]
-				t[prefix..k] = (val or 0) + v
+				if type(result) == 'table' then
+					merge(t, result, prefix..k..'.')
+				else
+					local val = t[prefix..k]
+					t[prefix..k] = (val or 0) + v
+				end
 			end
 		end
 	end
diff --git a/modules/meson.build b/modules/meson.build
index 38612254..5ffe95f8 100644
--- a/modules/meson.build
+++ b/modules/meson.build
@@ -7,6 +7,7 @@ lua_mod_src = [  # add lua modules without separate meson.build
   files('dns64/dns64.lua'),
   files('etcd/etcd.lua'),
   files('graphite/graphite.lua'),
+  files('prefetch/prefetch.lua'),
   files('predict/predict.lua'),
   files('prefill/prefill.lua'),
   files('priming/priming.lua'),
@@ -35,6 +36,13 @@ integr_tests += [
 mod_inc_dir = include_directories('..', '../contrib',
   luajit.get_pkgconfig_variable('includedir'))
 
+mod_deps = [
+  contrib_dep,
+  libknot,
+  libuv,
+  luajit,
+]
+
 # handle more complex C/LUA modules separately
 subdir('bogus_log')
 # cookies module is not currently used
diff --git a/modules/nsid/meson.build b/modules/nsid/meson.build
index 354e70b9..3c418bc9 100644
--- a/modules/nsid/meson.build
+++ b/modules/nsid/meson.build
@@ -9,10 +9,7 @@ c_src_lint += nsid_src
 nsid_mod = shared_module(
   'nsid',
   nsid_src,
-  dependencies: [
-    libknot,
-    luajit,
-  ],
+  dependencies: mod_deps,
   include_directories: mod_inc_dir,
   name_prefix: '',
   install: true,
diff --git a/modules/predict/README.rst b/modules/predict/README.rst
index 966c4ca4..0bc85671 100644
--- a/modules/predict/README.rst
+++ b/modules/predict/README.rst
@@ -2,34 +2,19 @@
 
 .. _mod-predict:
 
-Prefetching records
-===================
-
-The ``predict`` module helps to keep the cache hot by prefetching records.
-It can utilize two independent mechanisms to select the records which should be refreshed:
-expiring records and prediction.
-
-Expiring records
-----------------
-
-This mechanism is always active when the predict module is loaded and it is not configurable.
-
-Any time the resolver answers with records that are about to expire,
-they get refreshed. (see :c:func:`is_expiring`)
-That improves latency for records which get frequently queried, relatively to their TTL.
-
 Prediction
 ----------
 
-The predict module can also learn usage patterns and repetitive queries,
+``predict`` is an experimental module that tries to help keep the cache hot by prefetching records using a prediction mechanism to select records which should be refreshed.
+
+The module can learn usage patterns and repetitive queries,
 though this mechanism is a prototype and **not recommended** for use in production or with high traffic.
 
 For example, if it makes a query every day at 18:00,
 the resolver expects that it is needed by that time and prefetches it ahead of time.
 This is helpful to minimize the perceived latency and keeps the cache hot.
 
-You can disable prediction by configuring ``period = 0``.
-Otherwise it will load the required :ref:`stats <mod-stats>` module if not present,
+It will load the required :ref:`stats <mod-stats>` module if not present,
 and it will use its :func:`stats.frequent` table and clear it periodically.
 
 .. tip:: The tracking window and period length determine memory requirements. If you have a server with relatively fast query turnover, keep the period low (hour for start) and shorter tracking window (5 minutes). For personal slower resolver, keep the tracking window longer (i.e. 30 minutes) and period longer (a day), as the habitual queries occur daily. Experiment to get the best results.
diff --git a/modules/predict/predict.lua b/modules/predict/predict.lua
index 0117fd52..38fb7258 100644
--- a/modules/predict/predict.lua
+++ b/modules/predict/predict.lua
@@ -172,18 +172,4 @@ function predict.config(config)
 	predict.init()
 end
 
-predict.layer = {
-	-- Prefetch all expiring (sub-)queries immediately after the request finishes.
-	-- Doing that immediately is simplest and avoids creating (new) large bursts of activity.
-	finish = function (_, req)
-		local qrys = req.rplan.resolved
-		for i = 0, (tonumber(qrys.len) - 1) do -- size_t doesn't work for some reason
-			local qry = qrys.at[i]
-			if qry.flags.EXPIRING == true then
-				resolve(kres.dname2str(qry.sname), qry.stype, qry.sclass, {'NO_CACHE'})
-			end
-		end
-	end
-}
-
 return predict
diff --git a/modules/prefetch/README.rst b/modules/prefetch/README.rst
new file mode 100644
index 00000000..4d5a5e3e
--- /dev/null
+++ b/modules/prefetch/README.rst
@@ -0,0 +1,18 @@
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+.. _mod-prefetch:
+
+Expiring records
+----------------
+
+The ``prefetch`` module helps to keep the cache hot by prefetching expiring records.
+
+This mechanism is activated when the module is loaded and it is not configurable.
+
+.. code-block:: lua
+
+	modules.load('prefetch')
+
+
+Any time the resolver answers with records that are about to expire, they get refreshed. (see :c:func:`is_expiring`)
+That improves latency for records which get frequently queried, relatively to their TTL.
diff --git a/modules/prefetch/prefetch.lua b/modules/prefetch/prefetch.lua
new file mode 100644
index 00000000..673cf3e5
--- /dev/null
+++ b/modules/prefetch/prefetch.lua
@@ -0,0 +1,21 @@
+-- SPDX-License-Identifier: GPL-3.0-or-later
+-- Speculative prefetching for repetitive and soon-expiring records to reduce latency.
+-- @module prefetch
+local prefetch = {}
+
+
+prefetch.layer = {
+	-- Prefetch all expiring (sub-)queries immediately after the request finishes.
+	-- Doing that immediately is simplest and avoids creating (new) large bursts of activity.
+	finish = function (_, req)
+		local qrys = req.rplan.resolved
+		for i = 0, (tonumber(qrys.len) - 1) do -- size_t doesn't work for some reason
+			local qry = qrys.at[i]
+			if qry.flags.EXPIRING == true then
+				resolve(kres.dname2str(qry.sname), qry.stype, qry.sclass, {'NO_CACHE'})
+			end
+		end
+	end
+}
+
+return prefetch
diff --git a/modules/refuse_nord/meson.build b/modules/refuse_nord/meson.build
index 5142ded6..7dc8b888 100644
--- a/modules/refuse_nord/meson.build
+++ b/modules/refuse_nord/meson.build
@@ -13,7 +13,7 @@ c_src_lint += refuse_nord_src
 refuse_nord_mod = shared_module(
   'refuse_nord',
   refuse_nord_src,
-  dependencies: libknot,
+  dependencies: mod_deps,
   include_directories: mod_inc_dir,
   name_prefix: '',
   install: true,
diff --git a/modules/stats/meson.build b/modules/stats/meson.build
index cb4ccd68..e1f4a49c 100644
--- a/modules/stats/meson.build
+++ b/modules/stats/meson.build
@@ -14,9 +14,7 @@ integr_tests += [
 stats_mod = shared_module(
   'stats',
   stats_src,
-  dependencies: [
-    libknot,
-  ],
+  dependencies: mod_deps,
   include_directories: mod_inc_dir,
   name_prefix: '',
   install: true,
diff --git a/modules/stats/stats.c b/modules/stats/stats.c
index ca3a932c..a8a29de2 100644
--- a/modules/stats/stats.c
+++ b/modules/stats/stats.c
@@ -57,10 +57,18 @@ enum const_metric {
 };
 struct const_metric_elm {
 	const char *key;
+	const char *sup_key;
+	const char *sub_key;
 	size_t val;
 };
 static struct const_metric_elm const_metrics[] = {
-	#define X(a,b) [metric_ ## a ## _ ## b] = { #a "." #b, 0 },
+	#define X(a,b) \
+		[metric_ ## a ## _ ## b] = { \
+			.key = #a "." #b, \
+			.sup_key = #a, \
+			.sub_key = #b, \
+			.val = 0 \
+		},
 	CONST_METRICS(X)
 	#undef X
 };
@@ -359,8 +367,31 @@ static int list_entry(const char *key, uint32_t key_len, trie_val_t *val, void *
 	if (!key_matches_prefix(key, key_len, ctx->key_prefix, ctx->key_prefix_len))
 		return 0;
 	size_t number = (size_t) *val;
-	auto_free char *key_nt = strndup(key, key_len);
-	json_append_member(ctx->root, key_nt, json_mknumber(number));
+
+	uint32_t dot_index = 0;
+	for (uint32_t i = 0; i < key_len; i++) {
+		if (!key[i])
+			break;
+		if (key[i] == '.') {
+			dot_index = i;
+		}
+	}
+
+	if (dot_index) {
+		auto_free char *sup_key_nt = strndup(key, dot_index);
+		auto_free char *sub_key_nt = strndup(key + dot_index + 1, key_len - dot_index - 1);
+		JsonNode *sup = json_find_member(ctx->root, sup_key_nt);
+		if (!sup) {
+			sup = json_mkobject();
+			json_append_member(ctx->root, sup_key_nt, sup);
+		}
+		if (kr_fails_assert(sup))
+			return 0;
+		json_append_member(sup, sub_key_nt, json_mknumber(number));
+	} else {
+		auto_free char *key_nt = strndup(key, key_len);
+		json_append_member(ctx->root, key_nt, json_mknumber(number));
+	}
 	return 0;
 }
 
@@ -376,8 +407,15 @@ static char* stats_list(void *env, struct kr_module *module, const char *args)
 	size_t args_len = args ? strlen(args) : 0;
 	for (unsigned i = 0; i < metric_const_end; ++i) {
 		struct const_metric_elm *elm = &const_metrics[i];
-		if (!args || strncmp(elm->key, args, args_len) == 0) {
-			json_append_member(root, elm->key, json_mknumber(elm->val));
+		if (!args || strcmp(elm->sup_key, args) == 0) {
+			JsonNode *sup = json_find_member(root, elm->sup_key);
+			if (!sup) {
+				sup = json_mkobject();
+				json_append_member(root, elm->sup_key, sup);
+			}
+			if (kr_fails_assert(sup))
+				break;
+			json_append_member(sup, elm->sub_key, json_mknumber(elm->val));
 		}
 	}
 	struct list_entry_context ctx = {
diff --git a/modules/stats/test.integr/kresd_config.j2 b/modules/stats/test.integr/kresd_config.j2
index 872ce2e3..d0707691 100644
--- a/modules/stats/test.integr/kresd_config.j2
+++ b/modules/stats/test.integr/kresd_config.j2
@@ -9,32 +9,36 @@ FWD_TARGET = policy.FORWARD('192.0.2.1')
 function check_stats(got)
 	log_info(ffi.C.LOG_GRP_TESTS, 'checking if stat values match expected values:')
 	local expected = {
-		['answer.cd'] = 2,
-		['answer.cached'] = 1,
-		['answer.nodata'] = 1,
-		['answer.noerror'] = 2,
-		['answer.nxdomain'] = 1,
-		['answer.servfail'] = 2,
-		['answer.edns0'] = 6,
-		['answer.ra'] = 6,
-		['answer.rd'] = 5,
-		['answer.do'] = 1,
-		['answer.ad'] = 0,
-		['answer.tc'] = 0,
-		['answer.aa'] = 0,
-		['answer.total'] = 6
+		['answer'] = {
+			['cd'] = 2,
+			['cached'] = 1,
+			['nodata'] = 1,
+			['noerror'] = 2,
+			['nxdomain'] = 1,
+			['servfail'] = 2,
+			['edns0'] = 6,
+			['ra'] = 6,
+			['rd'] = 5,
+			['do'] = 1,
+			['ad'] = 0,
+			['tc'] = 0,
+			['aa'] = 0,
+			['total'] = 6
+		}
 	}
 	print(table_print(expected))
 
 	local ok = true
-	for key, expval in pairs(expected) do
-		if got[key] ~= expval then
-			log_info(ffi.C.LOG_GRP_TESTS,
-			    'ERROR: stats key ' .. key
-			    .. ' has unexpected value'
-			    .. ' (expected ' .. tostring(expval)
-			    .. ' got ' .. tostring(got[key] .. ')'))
-			ok = false
+	for sup_key, sup in pairs(expected) do
+		for sub_key, expval in pairs(sup) do
+			if got[sup_key][sub_key] ~= expval then
+				log_info(ffi.C.LOG_GRP_TESTS,
+				    'ERROR: stats key ' .. key
+				    .. ' has unexpected value'
+				    .. ' (expected ' .. tostring(expval)
+				    .. ' got ' .. tostring(got[key] .. ')'))
+				ok = false
+			end
 		end
 	end
 	if ok then