Merge branch 'master' into rrl-wip

14 files changed, 95 insertions, 72 deletions

diff --git a/lib/cache/api.c b/lib/cache/api.c
index 490f3d1c..0cd18534 100644
--- a/lib/cache/api.c
+++ b/lib/cache/api.c
@@ -175,7 +175,7 @@ int kr_cache_commit(struct kr_cache *cache)
 		return kr_error(EINVAL);
 	}
 	if (cache->api->commit) {
-		return cache_op(cache, commit, true);
+		return cache_op(cache, commit, true, true);
 	}
 	return kr_ok();
 }
diff --git a/lib/cache/cdb_api.h b/lib/cache/cdb_api.h
index bce8740f..23a795d5 100644
--- a/lib/cache/cdb_api.h
+++ b/lib/cache/cdb_api.h
@@ -58,10 +58,11 @@ struct kr_cdb_api {
 	int (*clear)(kr_cdb_pt db, struct kr_cdb_stats *stat);
 
 	/** Run after a row of operations to release transaction/lock if needed.
-	 * \param accept true=commit / false=abort
+	 * \param accept_rw whether the RW transaction should accept changes (commit vs. abort)
+	 * \param reset_ro whether the RO transaction should be ended (newest data next time)
 	 * \return error code - accepting RW transactions can fail with LMDB.
 	 */
-	int (*commit)(kr_cdb_pt db, struct kr_cdb_stats *stat, bool accept);
+	int (*commit)(kr_cdb_pt db, struct kr_cdb_stats *stat, bool accept_rw, bool reset_ro);
 
 	/* Data access */
 
diff --git a/lib/cache/cdb_lmdb.c b/lib/cache/cdb_lmdb.c
index 5351cd73..10611513 100644
--- a/lib/cache/cdb_lmdb.c
+++ b/lib/cache/cdb_lmdb.c
@@ -76,7 +76,7 @@ static inline kr_cdb_pt env2db(struct lmdb_env *env)
 	return (kr_cdb_pt)env;
 }
 
-static int cdb_commit(kr_cdb_pt db, struct kr_cdb_stats *stats, bool accept);
+static int cdb_commit(kr_cdb_pt db, struct kr_cdb_stats *stats, bool accept_rw, bool reset_ro);
 static void txn_abort(struct lmdb_env *env);
 
 /** @brief Convert LMDB error code. */
@@ -114,7 +114,7 @@ static inline MDB_val val_knot2mdb(knot_db_val_t v)
  * It's much lighter than reopen_env(). */
 static int refresh_mapsize(struct lmdb_env *env)
 {
-	int ret = cdb_commit(env2db(env), NULL, true);
+	int ret = cdb_commit(env2db(env), NULL, true, true);
 	if (!ret) ret = lmdb_error(env, mdb_env_set_mapsize(env->env, 0));
 	if (ret) return ret;
 
@@ -223,20 +223,20 @@ static int txn_get(struct lmdb_env *env, MDB_txn **txn, bool rdonly)
 	return kr_ok();
 }
 
-static int cdb_commit(kr_cdb_pt db, struct kr_cdb_stats *stats, bool accept)
+static int cdb_commit(kr_cdb_pt db, struct kr_cdb_stats *stats, bool accept_rw, bool reset_ro)
 {
 	struct lmdb_env *env = db2env(db);
-	if (!accept) {
-		txn_abort(env);
-		return kr_ok();
-	}
 
 	int ret = kr_ok();
 	if (env->txn.rw) {
-		if (stats) stats->commit++;
-		ret = lmdb_error(env, mdb_txn_commit(env->txn.rw));
+		if (accept_rw) {
+			if (stats) stats->commit++;
+			ret = lmdb_error(env, mdb_txn_commit(env->txn.rw));
+		} else {
+			mdb_txn_abort(env->txn.rw);
+		}
 		env->txn.rw = NULL; /* the transaction got freed even in case of errors */
-	} else if (env->txn.ro && env->txn.ro_active) {
+	} else if (reset_ro && env->txn.ro && env->txn.ro_active) {
 		mdb_txn_reset(env->txn.ro);
 		env->txn.ro_active = false;
 		env->txn.ro_curs_active = false;
@@ -256,7 +256,7 @@ static int txn_curs_get(struct lmdb_env *env, MDB_cursor **curs, struct kr_cdb_s
 	 * At least for rules we don't do the auto-commit feature. */
 	if (env->txn.rw) {
 		if (!env->is_cache) return kr_error(EINPROGRESS);
-		int ret = cdb_commit(env2db(env), stats, true);
+		int ret = cdb_commit(env2db(env), stats, true, false);
 		if (ret) return ret;
 	}
 	MDB_txn *txn = NULL;
@@ -312,7 +312,7 @@ static void cdb_close_env(struct lmdb_env *env, struct kr_cdb_stats *stats)
 
 	/* Get rid of any transactions. */
 	txn_free_ro(env);
-	cdb_commit(env2db(env), stats, env->is_cache);
+	cdb_commit(env2db(env), stats, env->is_cache, true);
 
 	mdb_env_sync(env->env, 1);
 	stats->close++;
@@ -574,7 +574,7 @@ static int cdb_clear(kr_cdb_pt db, struct kr_cdb_stats *stats)
 		if (ret == kr_ok()) {
 			ret = lmdb_error(env, mdb_drop(txn, env->dbi, 0));
 			if (ret == kr_ok() && env->is_cache) {
-				ret = cdb_commit(db, stats, true);
+				ret = cdb_commit(db, stats, true, true);
 			}
 			if (ret == kr_ok()) {
 				return ret;
@@ -588,7 +588,7 @@ static int cdb_clear(kr_cdb_pt db, struct kr_cdb_stats *stats)
 
 	/* We are about to switch to a different file, so end all txns, to be sure. */
 	txn_free_ro(env);
-	(void) cdb_commit(db, stats, env->is_cache);
+	(void)cdb_commit(db, stats, env->is_cache, true);
 
 	const char *path = NULL;
 	int ret = mdb_env_get_path(env->env, &path);
diff --git a/lib/defines.h b/lib/defines.h
index e8328928..24205896 100644
--- a/lib/defines.h
+++ b/lib/defines.h
@@ -73,11 +73,6 @@ static inline int KR_COLD kr_error(int x) {
 #define KR_DNAME_STR_MAXLEN (KNOT_DNAME_TXT_MAXLEN + 1)
 #define KR_RRTYPE_STR_MAXLEN (16 + 1)
 
-/* Compatibility with libknot<3.1.0 */
-#if KNOT_VERSION_HEX < 0x030100
-#define KNOT_EDNS_EDE_NONE (-1)
-#endif
-
 /*
  * Address sanitizer hints.
  */
diff --git a/lib/generic/lru.c b/lib/generic/lru.c
index 857b20b3..71b8730b 100644
--- a/lib/generic/lru.c
+++ b/lib/generic/lru.c
@@ -50,9 +50,9 @@ static uint item_size(const struct lru *lru, uint key_len, uint val_len)
 /** @internal Return pointer to value in an lru_item. */
 static void * item_val(const struct lru *lru, struct lru_item *it)
 {
-	size_t key_end = it->data + it->key_len - (char *)NULL;
+	size_t key_end = (uintptr_t)(it->data + it->key_len);
 	size_t val_begin = round_power(key_end, lru->val_alignment);
-	return (char *)NULL + val_begin;
+	return (void *)(uintptr_t)val_begin;
 }
 
 /** @internal Free each item. */
diff --git a/lib/generic/test_trie.c b/lib/generic/test_trie.c
index 9ecd67cd..ce164906 100644
--- a/lib/generic/test_trie.c
+++ b/lib/generic/test_trie.c
@@ -48,7 +48,7 @@ static void test_insert(void **state)
 		trie_val_t *data = trie_get_ins(t, dict[i], KEY_LEN(dict[i]));
 		assert_non_null(data);
 		assert_null(*data);
-		*data = (char *)NULL + i; // yes, ugly
+		*data = (void *)(intptr_t)i; // yes, ugly
 		assert_ptr_equal(trie_get_try(t, dict[i], KEY_LEN(dict[i])), data);
 	}
 	assert_int_equal(trie_weight(t), dict_size);
@@ -82,7 +82,7 @@ static void test_iter(void **state)
 		const char *key = trie_it_key(it, &len);
 		assert_int_equal(KEY_LEN(key), len);
 		assert_string_equal(key, dict_sorted[i]);
-		assert_ptr_equal(dict[(char *)*trie_it_val(it) - (char *)NULL],
+		assert_ptr_equal(dict[(uintptr_t)*trie_it_val(it)],
 				 dict_sorted[i]);
 	}
 	assert_true(trie_it_finished(it));
@@ -100,7 +100,7 @@ static void test_queue(void **state)
 		assert_non_null(key);
 		assert_int_equal(len, KEY_LEN(key));
 		assert_non_null(data);
-		ptrdiff_t key_i = (char *)*data - (char *)NULL;
+		uintptr_t key_i = (uintptr_t)*data;
 		assert_string_equal(key, dict[key_i]);
 
 		len = 30;
diff --git a/lib/generic/trie.c b/lib/generic/trie.c
index 21254eb4..e2ce061e 100644
--- a/lib/generic/trie.c
+++ b/lib/generic/trie.c
@@ -116,7 +116,7 @@ static inline void empty_root(node_t *root) {
 static void assert_portability(void) {
 #if FLAGS_HACK
 	kr_require(((union node){ .leaf = {
-			.key = (tkey_t *)(((uint8_t *)NULL) + 1),
+			.key = (tkey_t *)(void *)(uintptr_t)1,
 			.val = NULL
 		} }).branch.flags == 1);
 #endif
diff --git a/lib/layer/iterate.c b/lib/layer/iterate.c
index 6f312ca7..69fe344c 100644
--- a/lib/layer/iterate.c
+++ b/lib/layer/iterate.c
@@ -825,7 +825,10 @@ static int process_answer(knot_pkt_t *pkt, struct kr_request *req)
 		}
 	} else if (!query->parent) {
 		/* Answer for initial query */
-		const bool to_wire = ((pkt_class & (PKT_NXDOMAIN|PKT_NODATA)) != 0);
+		const bool to_wire = ((pkt_class & (PKT_NXDOMAIN|PKT_NODATA)) != 0)
+			/* We need to cover the case of positive wildcard answer
+			 * with over-limit NSEC3 iterations. */
+				|| query->flags.DNSSEC_WEXPAND;
 		state = pick_authority(pkt, req, to_wire);
 		if (state != kr_ok()) {
 			return KR_STATE_FAIL;
diff --git a/lib/resolve.c b/lib/resolve.c
index ec00b215..4730f105 100644
--- a/lib/resolve.c
+++ b/lib/resolve.c
@@ -246,11 +246,7 @@ static int pkt_padding(knot_pkt_t *packet, int32_t padding)
 	if (padding == -1) { /* use the default padding policy from libknot */
 		const size_t block_size = knot_wire_get_qr(packet->wire)
 					? KNOT_EDNS_ALIGNMENT_RESPONSE_DEFAULT
-				#if KNOT_VERSION_HEX < 0x030200
-					: KNOT_EDNS_ALIGNMENT_QUERY_DEFALT;
-				#else
 					: KNOT_EDNS_ALIGNMENT_QUERY_DEFAULT;
-				#endif
 		pad_bytes = knot_edns_alignment_size(packet->size, knot_rrset_size(opt_rr),
 							block_size);
 	}
diff --git a/lib/rules/api.c b/lib/rules/api.c
index 8e908a7a..5ecbe29e 100644
--- a/lib/rules/api.c
+++ b/lib/rules/api.c
@@ -141,9 +141,9 @@ int kr_rules_init_ensure(void)
 {
 	if (the_rules)
 		return kr_ok();
-	return kr_rules_init(NULL, 0);
+	return kr_rules_init(NULL, 0, true);
 }
-int kr_rules_init(const char *path, size_t maxsize)
+int kr_rules_init(const char *path, size_t maxsize, bool overwrite)
 {
 	if (the_rules)
 		return kr_error(EINVAL);
@@ -157,22 +157,17 @@ int kr_rules_init(const char *path, size_t maxsize)
 		// FIXME: the file will be sparse, but we still need to choose its size somehow.
 		// Later we might improve it to auto-resize in case of running out of space.
 		// Caveat: mdb_env_set_mapsize() can only be called without transactions open.
-		.maxsize = maxsize ? maxsize :
-			(size_t)(sizeof(size_t) > 4 ? 2048 : 500) * 1024*1024,
+		.maxsize = !overwrite ? 0 :
+			(maxsize ? maxsize : (size_t)(sizeof(size_t) > 4 ? 2048 : 500) * 1024*1024),
 	};
 	int ret = the_rules->api->open(&the_rules->db, &the_rules->stats, &opts, NULL);
-	/* No persistence - we always refill from config for now.
-	 * LATER:
-	 *  - Make it include versioning?
-	 *  - "\0stamp" key when loading config(s)?
-	 *  - Don't clear ruleset data that doesn't come directly from config;
-	 *    and add marks for that, etc.
-	 *    (after there actually are any kinds of rules like that)
-	 */
-	if (ret == 0) ret = ruledb_op(clear);
+
+	if (ret == 0 && overwrite) ret = ruledb_op(clear);
 	if (ret != 0) goto failure;
 	kr_require(the_rules->db);
 
+	if (!overwrite) return kr_ok(); // we assume that the caller ensured OK contents
+
 	ret = tag_names_default();
 	if (ret != 0) goto failure;
 
@@ -205,7 +200,13 @@ void kr_rules_deinit(void)
 int kr_rules_commit(bool accept)
 {
 	if (!the_rules) return kr_error(EINVAL);
-	return ruledb_op(commit, accept);
+	return ruledb_op(commit, accept, false);
+}
+
+int kr_rules_reset(void)
+{
+	if (!the_rules) return kr_error(EINVAL);
+	return ruledb_op(commit, false, true);
 }
 
 static bool kr_rule_consume_tags(knot_db_val_t *val, const struct kr_request *req)
@@ -817,16 +818,28 @@ int kr_rule_local_subtree(const knot_dname_t *apex, enum kr_rule_sub_t type,
 }
 
 
-/** Encode a subnet into a (longer) string.
+/** Encode a subnet into a (longer) string.  The result is in `buf` with returned length.
  *
  * The point is to have different encodings for different subnets,
  * with using just byte-length strings (e.g. for ::/1 vs. ::/2).
- * And we need to preserve order: FIXME description
- *  - natural partial order on subnets, one included in another
- *  - partial order on strings, one being a prefix of another
- *  - implies lexicographical order on the encoded strings
+ * You might imagine this as the space of all nodes of a binary trie.
  *
- * Consequently, given a set of subnets, the t
+ * == Key properties ==
+ * We're utilizing the order on the encoded strings.  LMDB uses lexicographical order.
+ * Optimization: the properties should cut down LMDB operation count when searching
+ * for rule sets typical in practice.  Some properties:
+ *  - full address is just a subnet containing only that address (/128 and /32)
+ *  - order of full addresses is kept the same as before encoding
+ *  - ancestor first: if subnet B is included inside subnet A, we get A < B
+ *  - subnet mixing: if two subnets do not share any address, all addresses of one
+ *    of them are ordered before all addresses of the other one
+ *
+ * == The encoding ==
+ * The encoding replaces each address bit by a pair of bits:
+ *  - 00 -> beyond the subnet's prefix
+ *  - 10 -> zero bit within the subnet's prefix
+ *  - 11 ->  one bit within the subnet's prefix
+ *  - we cut the byte-length - no need for all-zero suffixes
  */
 static int subnet_encode(const struct sockaddr *addr, int sub_len, uint8_t buf[32])
 {
@@ -837,17 +850,22 @@ static int subnet_encode(const struct sockaddr *addr, int sub_len, uint8_t buf[3
 		return kr_error(EINVAL);
 	const uint8_t *a = (const uint8_t *)/*sign*/kr_inaddr(addr);
 
-	// Algo: interleave bits of the address.  Bit pairs:
-	//  - 00 -> beyond the subnet's prefix
-	//  - 10 -> zero bit within the subnet's prefix
-	//  - 11 ->  one bit within the subnet's prefix
-	// Multiplying one uint8_t by 01010101 (in binary) will do interleaving.
 	int i;
 	// Let's hope that compiler optimizes this into something reasonable.
 	for (i = 0; sub_len > 0; ++i, sub_len -= 8) {
-		uint16_t x = a[i] * 85; // interleave by zero bits
-		uint8_t sub_mask = 255 >> (8 - MIN(sub_len, 8));
-		uint16_t r = x | (sub_mask * 85 * 2);
+		// r = a[i] interleaved by 1 bits (with 1s on the higher-value positions)
+		// https://graphics.stanford.edu/~seander/bithacks.html#Interleave64bitOps
+		// but we modify it slightly: no need for the 0x5555 mask (==0b0101010101010101)
+		// or the y-part - we instead just set all odd bits to 1s.
+		uint16_t r = (
+			(a[i] * 0x0101010101010101ULL & 0x8040201008040201ULL)
+				* 0x0102040810204081ULL >> 49
+		        ) | 0xAAAAU/* = 0b1010'1010'1010'1010 */;
+		// now r might just need clipping
+		if (sub_len < 8) {
+			uint16_t mask = 0xFFFFffffU << (2 * (8 - sub_len));
+			r &= mask;
+		}
 		buf[(ssize_t)2*i] = r / 256;
 		buf[(ssize_t)2*i + 1] = r % 256;
 	}
@@ -855,7 +873,7 @@ static int subnet_encode(const struct sockaddr *addr, int sub_len, uint8_t buf[3
 }
 
 // Is `a` subnet-prefix of `b`?  (a byte format of subnet_encode())
-bool subnet_is_prefix(uint8_t a, uint8_t b)
+static bool subnet_is_prefix(uint8_t a, uint8_t b)
 {
 	while (true) {
 		if (a >> 6 == 0)
diff --git a/lib/rules/api.h b/lib/rules/api.h
index 1069ef4d..f1737a19 100644
--- a/lib/rules/api.h
+++ b/lib/rules/api.h
@@ -19,11 +19,13 @@ typedef uint64_t kr_rule_tags_t;
 
 /** Open the rule DB.
  *
- * You can call this to override the path or size (NULL/0 -> default).
- * Not allowed if already open (EINVAL), so this optional call has to come
- * before writing anything into the DB. */
+ * You can call this to override the path or size (NULL/0 -> default)
+ * or choose not to overwrite the DB with just the defaults.
+ *
+ * \return error code.  Not allowed if already open (EINVAL),
+ * so this optional call has to come before writing anything into the DB. */
 KR_EXPORT
-int kr_rules_init(const char *path, size_t maxsize);
+int kr_rules_init(const char *path, size_t maxsize, bool overwrite);
 /** kr_rules_init() but OK if already open, and not allowing to override defaults. */
 KR_EXPORT
 int kr_rules_init_ensure(void);
@@ -36,10 +38,22 @@ void kr_rules_deinit(void);
  * Normally commit happens only on successfully loading a config file.
  * However, an advanced user may get in trouble e.g. if calling resolve() from there,
  * causing even an assertion failure.  In that case they might want to commit explicitly.
+ *
+ * If only read-only transaction is open, this will NOT reset it to the newest data.
  */
 KR_EXPORT
 int kr_rules_commit(bool accept);
 
+/** Reset to the latest version of rules committed in the DB.
+ *
+ * Note that this is not always a good idea.  For example, the `forward` rules
+ * now use data from both the DB and lua config, so reloading only the DB
+ * may lead to weird behavior in some cases.
+ * (Modifications will also do this, as you can only modify the latest DB.)
+ */
+KR_EXPORT
+int kr_rules_reset(void);
+
 /** Try answering the query from local data; WIP: otherwise determine data source overrides.
  *
  * \return kr_error() on errors, >0 if answered, 0 otherwise (also when forwarding)
diff --git a/lib/rules/impl.h b/lib/rules/impl.h
index 0d7de513..1a2ee4dd 100644
--- a/lib/rules/impl.h
+++ b/lib/rules/impl.h
@@ -20,7 +20,7 @@ extern struct kr_rules *the_rules;
 
 #define ENSURE_the_rules \
 	if (!the_rules) { \
-		int ret = kr_rules_init(NULL, 0); \
+		int ret = kr_rules_init(NULL, 0, true); \
 		if (ret) return ret; \
 	}
 
diff --git a/lib/selection.c b/lib/selection.c
index 9cdd1a60..cdef1701 100644
--- a/lib/selection.c
+++ b/lib/selection.c
@@ -173,7 +173,7 @@ int put_rtt_state(const uint8_t *ip, size_t len, struct rtt_state state,
 				.data = &state };
 
 	int ret = cache->api->write(db, stats, &key, &value, 1);
-	cache->api->commit(db, stats, true);
+	kr_cache_commit(cache);
 
 	free(key.data);
 	return ret;
diff --git a/lib/utils.c b/lib/utils.c
index 04b1bcb9..de7c02cb 100644
--- a/lib/utils.c
+++ b/lib/utils.c
@@ -1232,11 +1232,7 @@ char *kr_pkt_text(const knot_pkt_t *pkt)
 
 const knot_dump_style_t KR_DUMP_STYLE_DEFAULT = { /* almost all = false, */
 	.show_ttl = true,
-#if KNOT_VERSION_HEX >= 0x030200
 	.human_timestamp = true,
-#else
-	.human_tmstamp = true,
-#endif
 };
 
 char *kr_rrset_text(const knot_rrset_t *rr)