Merge !1403: tests, ci: fix and reintroduce dnstap tests + nits

12 files changed, 111 insertions, 73 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4d50a5eb..641de80a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -109,7 +109,7 @@ build:
     - meson build_ci --default-library=static --prefix=$PREFIX -Dmalloc=disabled -Dwerror=true -Dextra_tests=enabled
     - ninja -C build_ci
     - ninja -C build_ci install >/dev/null
-    - ${MESON_TEST} --suite unit --suite config --no-suite snowflake
+    - ${MESON_TEST} --suite unit --suite config --suite dnstap --no-suite snowflake
 
 build-knot32:
   <<: *build
@@ -118,7 +118,7 @@ build-knot32:
     - meson build_ci_knot32 --default-library=static --prefix=$PREFIX -Dmalloc=disabled -Dwerror=true -Dextra_tests=enabled
     - ninja -C build_ci_knot32
     - ninja -C build_ci_knot32 install >/dev/null
-    - ${MESON_TEST} --suite unit --suite config --no-suite snowflake
+    - ${MESON_TEST} --suite unit --suite config --suite dnstap --no-suite snowflake
 
 build-asan:
   <<: *build
@@ -129,7 +129,7 @@ build-asan:
     - ninja -C build_ci_asan install >/dev/null
       # TODO _leaks: not sure what exactly is wrong in leak detection on config tests
       # TODO skip_asan: all three of these disappear locally when using gcc 9.1 (except some leaks)
-    - MESON_TESTTHREADS=1 ASAN_OPTIONS=detect_leaks=0 ${MESON_TEST} --suite unit --suite config --no-suite skip_asan --no-suite snowflake
+    - MESON_TESTTHREADS=1 ASAN_OPTIONS=detect_leaks=0 ${MESON_TEST} --suite unit --suite config --suite dnstap --no-suite skip_asan --no-suite snowflake
 
 build:macOS:
   <<: *nodep
diff --git a/ci/images/README.md b/ci/images/README.md
index d9efe0e8..3d09f603 100644
--- a/ci/images/README.md
+++ b/ci/images/README.md
@@ -39,3 +39,11 @@ $ ./push.sh debian-11     # pushes the local image into target registry
 $ ./update.sh debian-11   # utility wrapper that both builds and pushes the image
 $ ./update.sh */          # use shell expansion of dirnames to update all images
 ```
+
+By default, a branch of Knot DNS deemed to be stable is selected according to
+the `vars.sh` file. To build an image for a different Knot DNS branch, set the
+`KNOT_BRANCH` environment variable to the name of the branch, e.g.:
+
+```
+$ KNOT_BRANCH='3.2' ./update.sh debian-11
+```
diff --git a/ci/images/build.sh b/ci/images/build.sh
index 39ee6171..1e9eabb5 100755
--- a/ci/images/build.sh
+++ b/ci/images/build.sh
@@ -9,5 +9,15 @@ if [ -n "$COVERITY_SCAN_TOKEN" ]; then
 	SECRETS="$SECRETS --secret id=coverity-token,env=COVERITY_SCAN_TOKEN"
 fi
 
+DOCKERFILE="$(realpath "${IMAGE}")/Dockerfile"
+
+cd "$CURRENT_DIR/../.."
 export DOCKER_BUILDKIT=1 # Enables using secrets in docker-build
-docker build --pull --no-cache -t "${FULL_NAME}" "${IMAGE}" --build-arg KNOT_BRANCH=${KNOT_BRANCH} $SECRETS
+docker build \
+	--pull \
+	--no-cache \
+	--tag "${FULL_NAME}" \
+	--file "${DOCKERFILE}" \
+	. \
+	--build-arg KNOT_BRANCH=${KNOT_BRANCH} \
+	$SECRETS
diff --git a/ci/images/debian-11/Dockerfile b/ci/images/debian-11/Dockerfile
index 59f170ba..9645cddb 100644
--- a/ci/images/debian-11/Dockerfile
+++ b/ci/images/debian-11/Dockerfile
@@ -21,10 +21,12 @@ RUN apt-get install -y -qqq git make cmake pkg-config meson \
 
 # Build and testing deps for Resolver's dnstap module (go stuff is just for testing)
 RUN apt-get install -y -qqq \
-	protobuf-c-compiler libprotobuf-c-dev libfstrm-dev
-# Maintaining the go stuff in CI really seems more trouble than worth.
-#	golang-any
-#RUN bash -c "go get github.com/{FiloSottile/gvt,cloudflare/dns,dnstap/golang-dnstap,golang/protobuf/proto}"
+	protobuf-c-compiler libprotobuf-c-dev libfstrm-dev \
+	golang-any
+COPY ./tests/dnstap /root/tests/dnstap
+WORKDIR /root/tests/dnstap/src/dnstap-test
+RUN go get .
+WORKDIR /root
 
 # documentation dependencies
 RUN apt-get install -y -qqq doxygen python3-sphinx python3-breathe python3-sphinx-rtd-theme
diff --git a/ci/images/debian-buster/Dockerfile b/ci/images/debian-buster/Dockerfile
index 4b47dda1..39f43277 100644
--- a/ci/images/debian-buster/Dockerfile
+++ b/ci/images/debian-buster/Dockerfile
@@ -25,9 +25,10 @@ RUN apt-get install -y -qqq git make cmake pkg-config meson \
 RUN apt-get install -y -qqq \
 	protobuf-c-compiler libprotobuf-c-dev libfstrm-dev \
 	golang-any
-# Some stuff won't work on buster:
-# package crypto/ed25519: unrecognized import path "crypto/ed25519"
-#RUN bash -c "go get github.com/{FiloSottile/gvt,cloudflare/dns,dnstap/golang-dnstap}"
+COPY ./tests/dnstap /root/tests/dnstap
+WORKDIR /root/tests/dnstap/src/dnstap-test
+RUN go get .
+WORKDIR /root
 
 # documentation dependencies
 RUN apt-get install -y -qqq doxygen python3-sphinx python3-breathe python3-sphinx-rtd-theme
@@ -41,7 +42,7 @@ RUN pip3 install pylint
 RUN pip3 install pep8
 RUN pip3 install pytest-xdist
 # tests/pytest dependencies: skip over broken versions
-RUN pip3 install 'dnspython != 2.0.0' jinja2 'pytest != 6.0.0' pytest-html pytest-xdist
+RUN pip3 install 'dnspython != 2.0.0' 'jinja2 == 2.11.3' 'pytest != 6.0.0' pytest-html pytest-xdist
 
 # packet capture tools for Deckard
 RUN apt-get install --no-install-suggests --no-install-recommends -y -qqq tcpdump wireshark-common
diff --git a/tests/dnstap/src/dnstap-test/go.mod b/tests/dnstap/src/dnstap-test/go.mod
new file mode 100644
index 00000000..562fd734
--- /dev/null
+++ b/tests/dnstap/src/dnstap-test/go.mod
@@ -0,0 +1,18 @@
+module gitlab.nic.cz/knot/knot-resolver/tests/dnstap-test
+
+go 1.17
+
+require (
+	github.com/cloudflare/dns v0.0.0-20151007113418-e20ffa3da443
+	github.com/dnstap/golang-dnstap v0.4.0
+	google.golang.org/protobuf v1.30.0
+)
+
+require (
+	github.com/FiloSottile/gvt v0.0.0-20180825041312-4899cb1641fb // indirect
+	github.com/farsightsec/golang-framestream v0.3.0 // indirect
+	github.com/miekg/dns v1.1.31 // indirect
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
+	golang.org/x/net v0.0.0-20190923162816-aa69164e4478 // indirect
+	golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe // indirect
+)
diff --git a/tests/dnstap/src/dnstap-test/go.sum b/tests/dnstap/src/dnstap-test/go.sum
new file mode 100644
index 00000000..5d602c23
--- /dev/null
+++ b/tests/dnstap/src/dnstap-test/go.sum
@@ -0,0 +1,52 @@
+github.com/FiloSottile/gvt v0.0.0-20180825041312-4899cb1641fb h1:Xi8CY7gyUpRyU9mXeR68CG5q6cgyhfOAo3hSmZNI6t0=
+github.com/FiloSottile/gvt v0.0.0-20180825041312-4899cb1641fb/go.mod h1:Jmwi7skQ6KCp/c3K39TAUuvnriIcEljm/rVSk0+gaBo=
+github.com/cloudflare/dns v0.0.0-20151007113418-e20ffa3da443 h1:dYR6/V5rx/uaHsy4m1JuWfKYZO0r+G89BLD+XN7s9AI=
+github.com/cloudflare/dns v0.0.0-20151007113418-e20ffa3da443/go.mod h1:pa4p3oKOxzbXjrV5AGD1v5xjL7skv9BvO4J0Llo3P+s=
+github.com/dnstap/golang-dnstap v0.4.0 h1:KRHBoURygdGtBjDI2w4HifJfMAhhOqDuktAokaSa234=
+github.com/dnstap/golang-dnstap v0.4.0/go.mod h1:FqsSdH58NAmkAvKcpyxht7i4FoBjKu8E4JUPt8ipSUs=
+github.com/farsightsec/golang-framestream v0.3.0 h1:/spFQHucTle/ZIPkYqrfshQqPe2VQEzesH243TjIwqA=
+github.com/farsightsec/golang-framestream v0.3.0/go.mod h1:eNde4IQyEiA5br02AouhEHCu3p3UzrCdFR4LuQHklMI=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo=
+github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
diff --git a/tests/dnstap/src/dnstap-test/main.go b/tests/dnstap/src/dnstap-test/main.go
index c04b4c14..a9d6635c 100644
--- a/tests/dnstap/src/dnstap-test/main.go
+++ b/tests/dnstap/src/dnstap-test/main.go
@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"github.com/cloudflare/dns"
 	dnstap "github.com/dnstap/golang-dnstap"
-	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/proto"
 	"io"
 	"io/ioutil"
 	"log"
diff --git a/tests/dnstap/src/dnstap-test/run.sh b/tests/dnstap/src/dnstap-test/run.sh
index 2f32ea18..37822b75 100755
--- a/tests/dnstap/src/dnstap-test/run.sh
+++ b/tests/dnstap/src/dnstap-test/run.sh
@@ -4,6 +4,7 @@ KRESD_CMD=$1
 MESON_BUILD_ROOT=$(pwd)
 mkdir -p tests/dnstap
 export GOPATH=$MESON_BUILD_ROOT/tests/dnstap
+echo "$GOPATH"
 cd "$(dirname $0)"
 DNSTAP_TEST=dnstap-test
 
@@ -11,15 +12,17 @@ if [ -z "$GITLAB_CI" ]; then
 	type -P go >/dev/null || exit 77
 	echo "Building the dnstap test and its dependencies..."
 	# some packages may be missing on the system right now
-	go get github.com/{FiloSottile/gvt,cloudflare/dns,dnstap/golang-dnstap,golang/protobuf/proto}
+	go get .
 else
 	# In CI we've prebuilt dependencies into the default GOPATH.
 	# We're in a scratch container, so we just add the dnstap test inside.
 	export GOPATH=/root/go
 fi
-DTAP=$GOPATH/src/$DNSTAP_TEST
+DTAP_DIR="$GOPATH/src"
+DTAP="$DTAP_DIR/$DNSTAP_TEST"
+mkdir -p "$DTAP_DIR"
 rm -f $DTAP && ln -s $(realpath ..)/$DNSTAP_TEST $DTAP
-go install $DNSTAP_TEST
+go install .
 
 
 CONFIG=$(realpath ./config)
diff --git a/tests/dnstap/src/dnstap-test/vendor/manifest b/tests/dnstap/src/dnstap-test/vendor/manifest
deleted file mode 100644
index 27c1dec1..00000000
--- a/tests/dnstap/src/dnstap-test/vendor/manifest
+++ /dev/null
@@ -1,55 +0,0 @@
-{
-	"version": 0,
-	"dependencies": [
-		{
-			"importpath": "github.com/cloudflare/dns",
-			"repository": "https://github.com/cloudflare/dns",
-			"vcs": "git",
-			"revision": "e20ffa3da443071c7b3d164dec5b1f80dfb2ecf3",
-			"branch": "master",
-			"notests": true
-		},
-		{
-			"importpath": "github.com/dnstap/golang-dnstap",
-			"repository": "https://github.com/dnstap/golang-dnstap",
-			"vcs": "git",
-			"revision": "0145fd8482619f9c04788c7ba4e96cdeef64a041",
-			"branch": "master",
-			"notests": true
-		},
-		{
-			"importpath": "github.com/farsightsec/golang-framestream",
-			"repository": "https://github.com/farsightsec/golang-framestream",
-			"vcs": "git",
-			"revision": "b600ccf606747139c84b6d69b5c3988164db4d42",
-			"branch": "master",
-			"notests": true
-		},
-		{
-			"importpath": "github.com/golang/protobuf/proto",
-			"repository": "https://github.com/golang/protobuf",
-			"vcs": "git",
-			"revision": "8ee79997227bf9b34611aee7946ae64735e6fd93",
-			"branch": "master",
-			"path": "/proto",
-			"notests": true
-		},
-		{
-			"importpath": "github.com/golang/protobuf/ptypes/any",
-			"repository": "https://github.com/golang/protobuf",
-			"vcs": "git",
-			"revision": "8ee79997227bf9b34611aee7946ae64735e6fd93",
-			"branch": "master",
-			"path": "ptypes/any",
-			"notests": true
-		},
-		{
-			"importpath": "github.com/miekg/dns",
-			"repository": "https://github.com/miekg/dns",
-			"vcs": "git",
-			"revision": "f4d2b086946a624202dc59e6a43f72e8f3f02bc1",
-			"branch": "master",
-			"notests": true
-		}
-	]
-}
\ No newline at end of file
diff --git a/tests/pytests/pylintrc b/tests/pytests/pylintrc
index 2c406be2..5e2b50b5 100644
--- a/tests/pytests/pylintrc
+++ b/tests/pytests/pylintrc
@@ -11,7 +11,6 @@ disable=
     line-too-long,  # checked by flake8
     invalid-name,
     broad-except,
-    bad-continuation,
     global-statement,
     no-else-return,
     redefined-outer-name,  # commonly used with pytest fixtures
diff --git a/tests/pytests/test_random_close.py b/tests/pytests/test_random_close.py
index a7cc8777..cadd8ef7 100644
--- a/tests/pytests/test_random_close.py
+++ b/tests/pytests/test_random_close.py
@@ -18,7 +18,7 @@ import utils
 QPS = 500
 
 
-def random_string(size=32, chars=(string.ascii_lowercase + string.digits)):
+def random_string(size=32, chars=string.ascii_lowercase + string.digits):
     return ''.join(random.choice(chars) for x in range(size))