journal: provide compress_threshold_bytes parameter

Previously the compression threshold was hardcoded to 512, which meant that
smaller values wouldn't be compressed. This left some storage savings on the
table, so instead, we make that number tunable.

1 files changed, 95 insertions, 7 deletions

diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c
index 517c9102a6..3b9917b4a8 100644
--- a/src/journal/test-journal.c
+++ b/src/journal/test-journal.c
@@ -43,7 +43,7 @@ static void test_non_empty(void) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f) == 0);
 
         dual_timestamp_get(&ts);
 
@@ -105,8 +105,8 @@ static void test_non_empty(void) {
 
         assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0);
 
-        journal_file_rotate(&f, true, true, NULL);
-        journal_file_rotate(&f, true, true, NULL);
+        journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
+        journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
 
         (void) journal_file_close(f);
 
@@ -132,13 +132,13 @@ static void test_empty(void) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, NULL, &f1) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f1) == 0);
 
-        assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &f2) == 0);
+        assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f2) == 0);
 
-        assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, NULL, &f3) == 0);
+        assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f3) == 0);
 
-        assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f4) == 0);
+        assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f4) == 0);
 
         journal_file_print_header(f1);
         puts("");
@@ -165,6 +165,91 @@ static void test_empty(void) {
         (void) journal_file_close(f4);
 }
 
+#if HAVE_XZ || HAVE_LZ4
+static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
+        dual_timestamp ts;
+        JournalFile *f;
+        struct iovec iovec;
+        Object *o;
+        uint64_t p;
+        char t[] = "/tmp/journal-XXXXXX";
+        char data[2048] = {0};
+        bool is_compressed;
+        int r;
+
+        assert_se(data_size <= sizeof(data));
+
+        log_set_max_level(LOG_DEBUG);
+
+        assert_se(mkdtemp(t));
+        assert_se(chdir(t) >= 0);
+
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0);
+
+        dual_timestamp_get(&ts);
+
+        iovec.iov_base = (void*) data;
+        iovec.iov_len = data_size;
+        assert_se(journal_file_append_entry(f, &ts, &iovec, 1, NULL, NULL, NULL) == 0);
+
+#if HAVE_GCRYPT
+        journal_file_append_tag(f);
+#endif
+        journal_file_dump(f);
+
+        /* We have to partially reimplement some of the dump logic, because the normal next_entry does the
+         * decompression for us. */
+        p = le64toh(f->header->header_size);
+        while (true) {
+                r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o);
+                assert_se(r == 0);
+                if (o->object.type == OBJECT_DATA)
+                        break;
+
+                assert_se(p < le64toh(f->header->tail_object_offset));
+                p = p + ALIGN64(le64toh(o->object.size));
+        }
+
+        is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0;
+
+        (void) journal_file_close(f);
+
+        log_info("Done...");
+
+        if (arg_keep)
+                log_info("Not removing %s", t);
+        else {
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
+
+                assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+        }
+
+        puts("------------------------------------------------------------");
+
+        return is_compressed;
+}
+
+static void test_min_compress_size(void) {
+        /* Note that XZ will actually fail to compress anything under 80 bytes, so you have to choose the limits
+         * carefully */
+
+        /* DEFAULT_MIN_COMPRESS_SIZE is 512 */
+        assert_se(!check_compressed((uint64_t) -1, 255));
+        assert_se(check_compressed((uint64_t) -1, 513));
+
+        /* compress everything */
+        assert_se(check_compressed(0, 96));
+        assert_se(check_compressed(8, 96));
+
+        /* Ensure we don't try to compress less than 8 bytes */
+        assert_se(!check_compressed(0, 7));
+
+        /* check boundary conditions */
+        assert_se(check_compressed(256, 256));
+        assert_se(!check_compressed(256, 255));
+}
+#endif
+
 int main(int argc, char *argv[]) {
         arg_keep = argc > 1;
 
@@ -174,6 +259,9 @@ int main(int argc, char *argv[]) {
 
         test_non_empty();
         test_empty();
+#if HAVE_XZ || HAVE_LZ4
+        test_min_compress_size();
+#endif
 
         return 0;
 }