tjh.dev
Clone of https://github.com/NixOS/nixpkgs.git (to stress-test knotserver)
nixos/vector: Add journald+ClickHouse test
+1
nixos/tests/vector/default.nix
+1
nixos/tests/vector/default.nix
···
8
8
file-sink = import ./file-sink.nix { inherit system pkgs; };
9
9
api = import ./api.nix { inherit system pkgs; };
10
10
dnstap = import ./dnstap.nix { inherit system pkgs; };
11
+
journald-clickhouse = import ./journald-clickhouse.nix { inherit system pkgs; };
11
12
nginx-clickhouse = import ./nginx-clickhouse.nix { inherit system pkgs; };
12
13
syslog-quickwit = import ./syslog-quickwit.nix { inherit system pkgs; };
13
14
}
+157
nixos/tests/vector/journald-clickhouse.nix
+157
nixos/tests/vector/journald-clickhouse.nix
···
1
+
import ../make-test-python.nix (
2
+
{ lib, pkgs, ... }:
3
+
let
4
+
# Take the original journald message and create a new payload which only
5
+
# contains the relevant fields - these must match the database columns.
6
+
journalVrlRemapTransform = {
7
+
journald_remap = {
8
+
inputs = [ "journald" ];
9
+
type = "remap";
10
+
source = ''
11
+
m = {}
12
+
m.app = .SYSLOG_IDENTIFIER
13
+
m.host = .host
14
+
m.severity = to_int(.PRIORITY) ?? 0
15
+
m.level = to_syslog_level(m.severity) ?? ""
16
+
m.message = strip_ansi_escape_codes!(.message)
17
+
m.timestamp = .timestamp
18
+
m.uid = to_int(._UID) ?? 0
19
+
m.pid = to_int(._PID) ?? 0
20
+
. = [m]
21
+
'';
22
+
};
23
+
};
24
+
in
25
+
{
26
+
name = "vector-journald-clickhouse";
27
+
meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
28
+
29
+
nodes = {
30
+
clickhouse =
31
+
{ config, pkgs, ... }:
32
+
{
33
+
virtualisation.diskSize = 5 * 1024;
34
+
virtualisation.memorySize = 4096;
35
+
36
+
networking.firewall.allowedTCPPorts = [ 6000 ];
37
+
38
+
services.vector = {
39
+
enable = true;
40
+
journaldAccess = true;
41
+
42
+
settings = {
43
+
sources = {
44
+
journald = {
45
+
type = "journald";
46
+
};
47
+
48
+
vector_source = {
49
+
type = "vector";
50
+
address = "[::]:6000";
51
+
};
52
+
};
53
+
54
+
transforms = journalVrlRemapTransform;
55
+
56
+
sinks = {
57
+
clickhouse = {
58
+
type = "clickhouse";
59
+
inputs = [
60
+
"journald_remap"
61
+
"vector_source"
62
+
];
63
+
endpoint = "http://localhost:8123";
64
+
database = "journald";
65
+
table = "logs";
66
+
date_time_best_effort = true;
67
+
};
68
+
};
69
+
};
70
+
71
+
};
72
+
73
+
services.clickhouse = {
74
+
enable = true;
75
+
};
76
+
};
77
+
78
+
vector =
79
+
{ config, pkgs, ... }:
80
+
{
81
+
services.vector = {
82
+
enable = true;
83
+
journaldAccess = true;
84
+
85
+
settings = {
86
+
sources = {
87
+
journald = {
88
+
type = "journald";
89
+
};
90
+
};
91
+
92
+
transforms = journalVrlRemapTransform;
93
+
94
+
sinks = {
95
+
vector_sink = {
96
+
type = "vector";
97
+
inputs = [ "journald_remap" ];
98
+
address = "clickhouse:6000";
99
+
};
100
+
};
101
+
};
102
+
};
103
+
};
104
+
};
105
+
106
+
testScript =
107
+
let
108
+
# work around quote/substitution complexity by Nix, Perl, bash and SQL.
109
+
databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS journald";
110
+
111
+
# https://clickhouse.com/blog/storing-log-data-in-clickhouse-fluent-bit-vector-open-telemetry
112
+
tableDDL = pkgs.writeText "table.sql" ''
113
+
CREATE TABLE IF NOT EXISTS journald.logs (
114
+
timestamp DateTime64(6),
115
+
app LowCardinality(String),
116
+
host LowCardinality(String),
117
+
level LowCardinality(String),
118
+
severity UInt8,
119
+
message String,
120
+
uid UInt16,
121
+
pid UInt32,
122
+
)
123
+
ENGINE = MergeTree()
124
+
ORDER BY (host, app, timestamp)
125
+
PARTITION BY toYYYYMM(timestamp)
126
+
'';
127
+
128
+
selectQuery = pkgs.writeText "select.sql" ''
129
+
SELECT COUNT(host) FROM journald.logs
130
+
WHERE message LIKE '%Vector has started%'
131
+
'';
132
+
in
133
+
''
134
+
clickhouse.wait_for_unit("clickhouse")
135
+
clickhouse.wait_for_open_port(6000)
136
+
clickhouse.wait_for_open_port(8123)
137
+
138
+
clickhouse.succeed(
139
+
"cat ${databaseDDL} | clickhouse-client"
140
+
)
141
+
142
+
clickhouse.succeed(
143
+
"cat ${tableDDL} | clickhouse-client"
144
+
)
145
+
146
+
for machine in clickhouse, vector:
147
+
machine.wait_for_unit("vector")
148
+
machine.wait_until_succeeds(
149
+
"journalctl -o cat -u vector.service | grep 'Vector has started'"
150
+
)
151
+
152
+
clickhouse.wait_until_succeeds(
153
+
"cat ${selectQuery} | clickhouse-client | grep 2"
154
+
)
155
+
'';
156
+
}
157
+
)