Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm

nginx log monitoring yay

+116
cozy-setup (move to another repo).md
··· 293 293 location /stub_status { 294 294 stub_status; 295 295 } 296 + location / { 297 + return 404; 298 + } 296 299 } 297 300 ``` 298 301 ··· 378 381 - [x] nginx: enable cache 379 382 - [x] nginx: rate-limit 380 383 - [ ] nginx: get metrics 384 + 385 + 386 + 387 + 388 + --- 389 + 390 + nginx cors for constellation + small burst bump 391 + 392 + ```nginx 393 + upstream cozy_constellation { 394 + server <tailnet ip>:6789; # bootes; ip so that we don't race on reboot with tailscale coming up, which nginx doesn't like 395 + keepalive 16; 396 + } 397 + 398 + server { 399 + server_name constellation.microcosm.blue; 400 + 401 + proxy_cache cozy_zone; 402 + proxy_cache_background_update on; 403 + proxy_cache_key "$scheme$proxy_host$uri$is_args$args$http_accept"; 404 + proxy_cache_lock on; # make simlutaneous requests for the same uri wait for it to appear in cache instead of hitting origin 405 + proxy_cache_lock_age 1s; 406 + proxy_cache_lock_timeout 2s; 407 + proxy_cache_valid 10s; # default -- should be explicitly set in the response headers 408 + proxy_cache_valid any 2s; # non-200s default 409 + proxy_read_timeout 5s; 410 + proxy_send_timeout 15s; 411 + proxy_socket_keepalive on; 412 + 413 + # take over cors responsibility from upsteram. `always` applies it to error responses. 414 + proxy_hide_header 'Access-Control-Allow-Origin'; 415 + proxy_hide_header 'Access-Control-Allowed-Methods'; 416 + proxy_hide_header 'Access-Control-Allow-Headers'; 417 + add_header 'Access-Control-Allow-Origin' '*' always; 418 + add_header 'Access-Control-Allow-Methods' 'GET' always; 419 + add_header 'Access-Control-Allow-Headers' '*' always; 420 + 421 + 422 + limit_req zone=cozy_ip_limit nodelay burst=150; 423 + limit_req zone=cozy_global_limit burst=1800; 424 + limit_req_status 429; 425 + 426 + location / { 427 + proxy_pass http://cozy_constellation; 428 + include proxy_params; 429 + proxy_http_version 1.1; 430 + proxy_set_header Connection ""; # for keepalive 431 + } 432 + 433 + 434 + listen 443 ssl; # managed by Certbot 435 + ssl_certificate /etc/letsencrypt/live/constellation.microcosm.blue/fullchain.pem; # managed by Certbot 436 + ssl_certificate_key /etc/letsencrypt/live/constellation.microcosm.blue/privkey.pem; # managed by Certbot 437 + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot 438 + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot 439 + 440 + } 441 + 442 + server { 443 + if ($host = constellation.microcosm.blue) { 444 + return 301 https://$host$request_uri; 445 + } # managed by Certbot 446 + 447 + 448 + server_name constellation.microcosm.blue; 449 + listen 80; 450 + return 404; # managed by Certbot 451 + } 452 + ``` 453 + 454 + re-reading about `nodelay`, i should probably remove it -- nginx would then queue requests to upstream, but still service them at the configured limit. it's fine for my internet since the global limit isn't nodelay, but probably less "fair" to clients if there's contention around the global limit (earlier requests would get all of theirs serviced before later ones can get in the queue) 455 + 456 + leaving it for now though. 457 + 458 + 459 + ### nginx logs to prom 460 + 461 + ```bash 462 + curl -LO https://github.com/martin-helmich/prometheus-nginxlog-exporter/releases/download/v1.11.0/prometheus-nginxlog-exporter_1.11.0_linux_amd64.deb 463 + apt install ./prometheus-nginxlog-exporter_1.11.0_linux_amd64.deb 464 + systemctl enable prometheus-nginxlog-exporter.service 465 + 466 + ``` 467 + 468 + have it run as www-data (maybe not the best idea but...) 469 + file `/usr/lib/systemd/system/prometheus-nginxlog-exporter.service` 470 + set User under service and remove capabilities bounding 471 + 472 + ```systemd 473 + User=www-data 474 + #CapabilityBoundingSet= 475 + ``` 476 + 477 + in `nginx.conf` in `http`: 478 + 479 + ```nginx 480 + log_format constellation_format "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""; 481 + ``` 482 + 483 + in `sites-available/constellation.microcosm.blue` in `server`: 484 + 485 + ```nginx 486 + # log format must match prometheus-nginx-log-exporter 487 + access_log /var/log/nginx/constellation-access.log constellation_format; 488 + ``` 489 + 490 + config at `/etc/prometheus-nginxlog-exporter.hcl` 491 + 492 + 493 + 494 + ```bash 495 + systemctl start prometheus-nginxlog-exporter.service 496 + ```