diff --git a/.dockerignore b/.dockerignore index 4f1d395d2..7f493b8f8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,102 @@ -**/.dockerignore +# Dependencies **/node_modules -**/npm-debug.log -**/README.md +node_modules +**/.pnpm-store +.pnpm-store + +# Build outputs **/.next +**/.turbo +**/dist +**/build +**/.nuxt +**/.output +**/.cache + +# Environment files +**/.env +**/.env.* +.env +.env.* +!.env.docker.example + +# Development files +**/.vscode +**/.idea +**/.DS_Store +.DS_Store +**/*.log +**/npm-debug.log +**/yarn-debug.log +**/yarn-error.log +**/pnpm-debug.log + +# Testing +**/coverage +**/.nyc_output +**/test-results +**/__tests__ +**/tests +**/*.test.ts +**/*.test.tsx +**/*.test.js +**/*.spec.ts +**/*.spec.tsx +**/*.spec.js + +# Git **/.git -node_modules -openstatus.db \ No newline at end of file +**/.gitignore +**/.gitattributes +.git +.gitignore + +# Documentation +**/README.md +**/CHANGELOG.md +**/LICENSE +**/*.md +!packages/**/README.md + +# CI/CD +**/.github +.github +**/.gitlab +**/.circleci +**/.drone.yml +**/.travis.yml + +# Docker +**/.dockerignore +**/Dockerfile +**/docker-compose*.yml +**/docker-compose*.yaml +.dockerignore +docker-compose*.yml +docker-compose*.yaml + +# Database files +openstatus.db +**/*.db +**/*.db-shm +**/*.db-wal + +# Temporary files +**/tmp +**/temp +**/.temp +**/.tmp + +# IDE +**/.vscode +**/.idea +**/.fleet + +# OS +.DS_Store +**/Thumbs.db + +# Other build artifacts +**/.contentlayer +**/public/build +**/.svelte-kit \ No newline at end of file diff --git a/.env.docker.example b/.env.docker.example new file mode 100644 index 000000000..610c649b8 --- /dev/null +++ b/.env.docker.example @@ -0,0 +1,153 @@ +# ============================================================================ +# Docker Environment Configuration (Example) +# ============================================================================ +# Copy this file to .env.docker and replace with your actual values +# +# NOTE: Build-time placeholders are set in each service's dofigen.yml file. +# Values in this file override those placeholders at runtime via docker compose. +# Required values are marked with [REQUIRED] + +# DATABASE +# ============================================================================ +# LibSQL database connection - points to the libsql container +DATABASE_URL=http://libsql:8080 +DATABASE_AUTH_TOKEN= + +# ClickHouse (optional - for advanced analytics) +CLICKHOUSE_URL= +CLICKHOUSE_USERNAME= +CLICKHOUSE_PASSWORD= + +# REDIS & QUEUE +# ============================================================================ +# Redis (optional) - for caching +UPSTASH_REDIS_REST_URL=http://localhost:6379 +UPSTASH_REDIS_REST_TOKEN=placeholder + +# QStash (optional - for background jobs) +QSTASH_CURRENT_SIGNING_KEY= +QSTASH_NEXT_SIGNING_KEY= +QSTASH_TOKEN= +QSTASH_URL=https://qstash.upstash.io/v1/publish/ + +# ANALYTICS +# ============================================================================ +# Tinybird (optional - for monitor analytics and charts) +# Leave empty to disable analytics features +# Get from: https://www.tinybird.co +TINY_BIRD_API_KEY= + +# Tinybird URL (optional - defaults to cloud API) +# For local Tinybird container, set to: http://tinybird-local:7181 +TINYBIRD_URL= + +# EMAIL +# ============================================================================ +# [REQUIRED] Resend API key for sending magic link emails +# Get from: https://resend.com +RESEND_API_KEY=re_your_resend_api_key_here + +# AUTHENTICATION +# ============================================================================ +# [REQUIRED] NextAuth secret - generate with: openssl rand -base64 32 +AUTH_SECRET=your-random-secret-here-min-32-chars + +# [REQUIRED] Self-hosted mode - enables magic link authentication +# Set to "true" to allow email login without OAuth +SELF_HOST=true + +# GitHub OAuth (optional) +# Get from: https://github.com/settings/developers +AUTH_GITHUB_ID= +AUTH_GITHUB_SECRET= + +# Google OAuth (optional) +# Get from: https://console.cloud.google.com +AUTH_GOOGLE_ID= +AUTH_GOOGLE_SECRET= + +# GOOGLE CLOUD +# ============================================================================ +# Google Cloud Platform (optional - for scheduled tasks) +GCP_PROJECT_ID= +GCP_LOCATION= +GCP_CLIENT_EMAIL= +GCP_PRIVATE_KEY= + +# Cron secret for scheduled jobs +CRON_SECRET=your-random-cron-secret + +# API KEYS +# ============================================================================ +# Unkey (optional - for API key management) +# Get from: https://unkey.dev +UNKEY_API_ID= +UNKEY_TOKEN= + +# Super admin token for privileged operations +SUPER_ADMIN_TOKEN= + +# Server region identifier +FLY_REGION=self-hosted + +# STRIPE +# ============================================================================ +# Stripe (optional - for payments) +# Get from: https://stripe.com/docs/keys +STRIPE_SECRET_KEY= +STRIPE_WEBHOOK_SECRET_KEY= +NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY= + +# VERCEL +# ============================================================================ +# Vercel (optional - for custom domains) +# Get from: https://vercel.com +PROJECT_ID_VERCEL= +TEAM_ID_VERCEL= +VERCEL_AUTH_BEARER_TOKEN= + +# Vercel Blob storage (optional) +BLOB_READ_WRITE_TOKEN= + +# OBSERVABILITY +# ============================================================================ +# Sentry (optional - error tracking) +# Get from: https://sentry.io +NEXT_PUBLIC_SENTRY_DSN= +SENTRY_AUTH_TOKEN= + +# OpenPanel (optional - analytics) +# Get from: https://openpanel.dev +NEXT_PUBLIC_OPENPANEL_CLIENT_ID= +OPENPANEL_CLIENT_SECRET= + +# PagerDuty (optional - alerting) +PAGERDUTY_APP_ID= + +# Slack webhook (optional) +SLACK_SUPPORT_WEBHOOK_URL= + +# SERVICE CONFIGURATION +# ============================================================================ +NODE_ENV=production + +# [REQUIRED] Public URL for the application +NEXT_PUBLIC_URL=http://localhost:3002 + +# Screenshot service (optional) +SCREENSHOT_SERVICE_URL= + +# External services +OPENSTATUS_INGEST_URL=https://openstatus-private-location.fly.dev + +# DEVELOPMENT & TESTING +# ============================================================================ +# Turbo build mode +TURBO_ENV_MODE=loose + +# Playground API keys (optional) +PLAYGROUND_UNKEY_API_KEY= + +# Workspace settings (optional) +WORKSPACES_LOOKBACK_30= +WORKSPACES_HIDE_URL= diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 000000000..14e20ef72 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,276 @@ +# Docker Setup Guide + +Complete guide for running OpenStatus with Docker + +## Quick Start + +```bash +# 1. Copy environment file +cp .env.docker.example .env.docker + +# 2. Configure required variables (see Configuration section) +vim .env.docker + +# 3. Build and start services +export DOCKER_BUILDKIT=1 +docker compose up -d + +# 4. Check service health +docker compose ps + +# 5. Run database migrations (required) +cd packages/db +pnpm migrate + +# 6. Deploy Tinybird local +cd packages/tinybird +tb --local deploy + +# 7. Seed database with test data (optional) +cd packages/db +pnpm seed + + +# 8. Access the application +open http://localhost:3002 # Dashboard +open http://localhost:3003 # Status Pages +``` + +## Cleanup + +```bash +# Remove stopped containers +docker compose down + +# Remove volumes +docker compose down -v + +# Clean build cache +docker builder prune +``` + +## Services + +| Service | Port | Purpose | +|---------|------|---------| +| workflows | 3000 | Background jobs | +| server | 3001 | API backend (tRPC) | +| dashboard | 3002 | Admin interface | +| status-page | 3003 | Public status pages | +| private-location | 8081 | Monitoring agent | +| libsql | 8080 | Database (HTTP) | +| libsql | 5001 | Database (gRPC) | +| tinybird-local | 7181 | Analytics | + + +## Architecture + +``` +┌─────────────┐ ┌─────────────┐ +│ Dashboard │────▶│ Server │ +│ (Next.js) │ │ (Bun) │ +└─────────────┘ └─────────────┘ + │ │ + ▼ ▼ +┌─────────────┐ ┌─────────────┐ +│ Status Page │ │ Workflows │ +│ (Next.js) │ │ (Bun) │ +└─────────────┘ └─────────────┘ + │ │ + └────────┬───────────┘ + ▼ + ┌─────────────┐ + │ LibSQL │ + │ (Database) │ + └─────────────┘ +``` + +## Database Setup + +The LibSQL container starts with an empty database. You must run migrations before using the application: + +```bash +cd packages/db +pnpm migrate +``` + +### Seeding Test Data (Optional) + +For development, you can populate the database with sample data: + +```bash +cd packages/db +pnpm seed +``` + +This creates: +- 3 workspaces (`love-openstatus`, `test2`, `test3`) +- 5 sample monitors and 1 status page +- Test user account: `ping@openstatus.dev` +- Sample incidents, status reports, and maintenance windows + +**Accessing Seeded Data:** + +To view the seeded data in the dashboard, you must log in using the seeded test email: + +1. Navigate to http://localhost:3002/login +2. Use magic link authentication with email: `ping@openstatus.dev` +3. Check your console/logs for the magic link +4. After logging in, you'll be in the `love-openstatus` workspace with all seeded data + +**If you use a different email address**, the system will create a new empty workspace for you. To access seeded data with a different account: + +1. Add your user to the seeded workspace using SQL: + ```bash + # First, find your user_id + curl -X POST http://localhost:8080/ -H "Content-Type: application/json" \ + -d '{"statements":["SELECT id, email FROM user"]}' + + # Then add association (replace USER_ID with your id) + curl -X POST http://localhost:8080/ -H "Content-Type: application/json" \ + -d '{"statements":["INSERT INTO users_to_workspaces (user_id, workspace_id, role) VALUES (USER_ID, 1, '\''owner'\'')"]}' + ``` + +2. Switch to the `love-openstatus` workspace using the workspace switcher in the dashboard sidebar + +## Tinybird Setup (Optional) + +Tinybird is used for analytics and monitoring metrics. The application will work without it, but analytics features will be unavailable. + +If you want to enable analytics, you can: +1. Use Tinybird Cloud and configure `TINY_BIRD_API_KEY` in `.env.docker` +2. Manually configure Tinybird Local (requires additional setup beyond this guide) + +## Configuration + +### Required Environment Variables + +Edit `.env.docker` and set: + +```bash +# Authentication +AUTH_SECRET=your-secret-here + +# Database +DATABASE_URL=http://libsql:8080 +DATABASE_AUTH_TOKEN=basic:token + +# Email +RESEND_API_KEY=test +``` + +### Optional Services + +Configure these for full functionality: + +```bash +# Redis +UPSTASH_REDIS_REST_URL= +UPSTASH_REDIS_REST_TOKEN= + +# Analytics +TINY_BIRD_API_KEY= + +# OAuth providers +AUTH_GITHUB_ID= +AUTH_GITHUB_SECRET= +AUTH_GOOGLE_ID= +AUTH_GOOGLE_SECRET= +``` + +See [.env.docker.example](.env.docker.example) for complete list. + +## Development Workflow + +### Common Commands + +```bash +# View logs +docker compose logs -f [service-name] + +# Restart service +docker compose restart [service-name] + +# Rebuild after code changes +docker compose up -d --build [service-name] + +# Stop all services +docker compose down + +# Reset database (removes all data) +docker compose down -v +# After resetting, re-run migrations: +# cd packages/db && pnpm migrate +``` + +### Authentication + +**Magic Link**: + +Set `SELF_HOST=true` in `.env.docker` to enable email-based magic link authentication. This allows users to sign in without configuring OAuth providers. + +**OAuth Providers**: + +Configure GitHub/Google OAuth credentials in `.env.docker` and set up callback URLs: + - GitHub: `http://localhost:3002/api/auth/callback/github` + - Google: `http://localhost:3002/api/auth/callback/google` + +### Creating Status Pages + +**Via Dashboard (Recommended)**: +1. Login to http://localhost:3002 +2. Create a workspace +3. Create a status page with a slug +4. Access at http://localhost:3003/[slug] + +**Via Database (Testing)**: +```bash +# Insert test data +curl -s http://localhost:8080/v2/pipeline \ + -H 'Content-Type: application/json' \ + --data-raw '{ + "requests":[{ + "type":"execute", + "stmt":{ + "sql":"INSERT INTO workspace (id, slug, name) VALUES (1, '\''test'\'', '\''Test Workspace'\'');" + } + }] + }' +``` + +### Resource Limits + +Add to `docker-compose.yaml`: + +```yaml +services: + dashboard: + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G + reservations: + cpus: '0.5' + memory: 512M +``` + +## Monitoring + +### Health Checks + +All services have automated health checks: + +```bash +# View health status +docker compose ps + +# Inspect specific service +docker inspect openstatus-dashboard --format='{{.State.Health.Status}}' +``` + +## Getting Help + +- **Documentation**: [docs.openstatus.dev](https://docs.openstatus.dev) +- **Discord**: [openstatus.dev/discord](https://www.openstatus.dev/discord) +- **GitHub Issues**: [github.com/openstatusHQ/openstatus/issues](https://github.com/openstatusHQ/openstatus/issues) diff --git a/README.md b/README.md index 335b50115..c11948feb 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,24 @@ Made with [Contrib.rocks](https://contrib.rocks) ## Getting Started 🚀 +### With Docker (Recommended) + +The fastest way to get started for both development and self-hosting: + +```sh +# 1. Copy environment file +cp .env.docker.example .env.docker + +# 2. Start all services +docker compose up -d + +# 3. Access the application +open http://localhost:3002 # Dashboard +open http://localhost:3003 # Status Pages +``` + +📖 **Full guide**: [DOCKER.md](DOCKER.md) | **Production deployment**: [DOCKER_PRODUCTION.md](DOCKER_PRODUCTION.md) + ### With Devbox You can use [Devbox](https://www.jetify.com/devbox/) and get started with the following commands: @@ -92,16 +110,16 @@ You can use [Devbox](https://www.jetify.com/devbox/) and get started with the fo devbox services up ``` -Alternatively, follow the instructions below. +### Manual Setup -### Requirements +#### Requirements - [Node.js](https://nodejs.org/en/) >= 20.0.0 - [pnpm](https://pnpm.io/) >= 8.6.2 - [Bun](https://bun.sh/) - [Turso CLI](https://docs.turso.tech/quickstart) -### Setup +#### Setup 1. Clone the repository diff --git a/apps/checker/cmd/private/main.go b/apps/checker/cmd/private/main.go index 391444793..1c6f5f759 100644 --- a/apps/checker/cmd/private/main.go +++ b/apps/checker/cmd/private/main.go @@ -66,12 +66,12 @@ func getEnv(key, fallback string) string { return fallback } -// UpdateMonitors fetches the latest monitors and starts/stops jobs as needed - func getClient(apiKey string) v1.PrivateLocationServiceClient { + ingestUrl := getEnv("OPENSTATUS_INGEST_URL", "https://openstatus-private-location.fly.dev") + client := v1.NewPrivateLocationServiceClient( http.DefaultClient, - "https://openstatus-private-location.fly.dev", + ingestUrl, connect.WithHTTPGet(), connect.WithInterceptors(NewAuthInterceptor(apiKey)), ) diff --git a/apps/checker/pkg/scheduler/scheduler.go b/apps/checker/pkg/scheduler/scheduler.go index cd59fad0c..0ab4dc002 100644 --- a/apps/checker/pkg/scheduler/scheduler.go +++ b/apps/checker/pkg/scheduler/scheduler.go @@ -29,6 +29,7 @@ type MonitorManager struct { mu sync.Mutex } +// UpdateMonitors fetches the latest monitors and starts/stops jobs as needed func (mm *MonitorManager) UpdateMonitors(ctx context.Context) { res, err := mm.Client.Monitors(ctx, &connect.Request[v1.MonitorsRequest]{}) if err != nil { @@ -100,8 +101,6 @@ func (mm *MonitorManager) UpdateMonitors(ctx context.Context) { } - - // TCP monitors: start jobs for new monitors for _, m := range res.Msg.TcpMonitors { currentIDs[m.Id] = struct{}{} @@ -175,7 +174,7 @@ func (mm *MonitorManager) UpdateMonitors(ctx context.Context) { } resp, ingestErr := mm.Client.IngestDNS(ctx.Context, &connect.Request[v1.IngestDNSRequest]{ Msg: &v1.IngestDNSRequest{ - MonitorId: monitor.Id, + MonitorId: monitor.Id, // Id: data.ID, // Uri: monitor.Uri, @@ -206,7 +205,6 @@ func (mm *MonitorManager) UpdateMonitors(ctx context.Context) { } } - for id := range mm.Scheduler.Tasks() { if _, stillExists := currentIDs[id]; !stillExists { mm.Scheduler.Del(id) diff --git a/apps/checker/pkg/tinybird/client.go b/apps/checker/pkg/tinybird/client.go index 5179fb74e..f138c36bc 100644 --- a/apps/checker/pkg/tinybird/client.go +++ b/apps/checker/pkg/tinybird/client.go @@ -7,11 +7,19 @@ import ( "fmt" "net/http" "net/url" + "os" "github.com/rs/zerolog/log" ) -const baseURL = "https://api.tinybird.co/v0/events" +func getBaseURL() string { + // Use local Tinybird container if available (Docker/self-hosted) + // https://www.tinybird.co/docs/api-reference + if tinybirdURL := os.Getenv("TINYBIRD_URL"); tinybirdURL != "" { + return tinybirdURL + "/v0/events" + } + return "https://api.tinybird.co/v0/events" +} type Client interface { SendEvent(ctx context.Context, event any, dataSourceName string) error @@ -20,17 +28,19 @@ type Client interface { type client struct { httpClient *http.Client apiKey string + baseURL string } func NewClient(httpClient *http.Client, apiKey string) Client { return client{ httpClient: httpClient, apiKey: apiKey, + baseURL: getBaseURL(), } } func (c client) SendEvent(ctx context.Context, event any, dataSourceName string) error { - requestURL, err := url.Parse(baseURL) + requestURL, err := url.Parse(c.baseURL) if err != nil { log.Ctx(ctx).Error().Err(err).Msg("unable to parse url") return fmt.Errorf("unable to parse url: %w", err) diff --git a/apps/dashboard/.dockerignore b/apps/dashboard/.dockerignore new file mode 100644 index 000000000..067cbb8dd --- /dev/null +++ b/apps/dashboard/.dockerignore @@ -0,0 +1,3 @@ +# This file is generated by Dofigen v2.5.1 +# See https://github.com/lenra-io/dofigen + diff --git a/apps/dashboard/.env.example b/apps/dashboard/.env.example index e137ee2d4..ac759e2cc 100644 --- a/apps/dashboard/.env.example +++ b/apps/dashboard/.env.example @@ -50,7 +50,7 @@ GCP_CLIENT_EMAIL= GCP_PRIVATE_KEY= CRON_SECRET= -EXTERNAL_API_URL= +VERCEL_URL=https://app.openstatus.dev PLAYGROUND_UNKEY_API_KEY= diff --git a/apps/dashboard/Dockerfile b/apps/dashboard/Dockerfile new file mode 100644 index 000000000..593691abc --- /dev/null +++ b/apps/dashboard/Dockerfile @@ -0,0 +1,83 @@ +# syntax=docker/dockerfile:1.11 +# This file is generated by Dofigen v2.5.1 +# See https://github.com/lenra-io/dofigen + +# builder +FROM node@sha256:0afb7822fac7bf9d7c1bf3b6e6c496dee6b2b64d8dfa365501a3c68e8eba94b2 AS builder +LABEL \ + org.opencontainers.image.base.digest="sha256:0afb7822fac7bf9d7c1bf3b6e6c496dee6b2b64d8dfa365501a3c68e8eba94b2" \ + org.opencontainers.image.base.name="docker.io/node:24-slim" +ENV \ + NEXT_PUBLIC_OPENPANEL_CLIENT_ID="test" \ + TINY_BIRD_API_KEY="test" \ + DATABASE_AUTH_TOKEN="test" \ + PNPM_HOME="/pnpm" \ + VERCEL_AUTH_BEARER_TOKEN="test" \ + OPENPANEL_CLIENT_SECRET="test" \ + RESEND_API_KEY="test" \ + AUTH_SECRET="build-time-placeholder-min-32-chars-long" \ + STRIPE_SECRET_KEY="test" \ + TEAM_ID_VERCEL="test" \ + CRON_SECRET="test" \ + PATH="$PNPM_HOME:$PATH" \ + UPSTASH_REDIS_REST_TOKEN="test" \ + UNKEY_API_ID="test" \ + DATABASE_URL="http://libsql:8080" \ + NEXT_PUBLIC_URL="http://localhost:3002" \ + NODE_ENV="production" \ + UPSTASH_REDIS_REST_URL="test" \ + UNKEY_TOKEN="test" \ + PROJECT_ID_VERCEL="test" +WORKDIR /app +COPY \ + --link \ + "." "/app/" +RUN <
- {process.env.NODE_ENV === "development" ? ( + {process.env.NODE_ENV === "development" || + process.env.SELF_HOST === "true" ? (
diff --git a/apps/dashboard/src/lib/auth/index.ts b/apps/dashboard/src/lib/auth/index.ts index f76747b7c..2d1852978 100644 --- a/apps/dashboard/src/lib/auth/index.ts +++ b/apps/dashboard/src/lib/auth/index.ts @@ -16,7 +16,7 @@ export const { handlers, signIn, signOut, auth } = NextAuth({ // debug: true, adapter, providers: - process.env.NODE_ENV === "development" + process.env.NODE_ENV === "development" || process.env.SELF_HOST === "true" ? [GitHubProvider, GoogleProvider, ResendProvider] : [GitHubProvider, GoogleProvider], callbacks: { diff --git a/apps/dashboard/src/lib/trpc/server.tsx b/apps/dashboard/src/lib/trpc/server.tsx index 33cb12c65..44769136a 100644 --- a/apps/dashboard/src/lib/trpc/server.tsx +++ b/apps/dashboard/src/lib/trpc/server.tsx @@ -31,7 +31,7 @@ export const trpc = createTRPCOptionsProxy({ headers: { "x-trpc-source": "server", }, - async fetch(url, options) { + fetch: async (url, options) => { const cookieStore = await cookies(); return fetch(url, { ...options, diff --git a/apps/dashboard/src/lib/trpc/shared.ts b/apps/dashboard/src/lib/trpc/shared.ts index a3290046c..8adeb357b 100644 --- a/apps/dashboard/src/lib/trpc/shared.ts +++ b/apps/dashboard/src/lib/trpc/shared.ts @@ -6,10 +6,9 @@ import superjson from "superjson"; const getBaseUrl = () => { if (typeof window !== "undefined") return ""; - const vc = process.env.VERCEL_URL; - // if (vc) return `https://${vc}`; - if (vc) return "https://app.openstatus.dev"; - return "http://localhost:3000"; + // Note: dashboard has its own tRPC API routes + if (process.env.VERCEL_URL) return "https://app.openstatus.dev"; // Vercel + return "http://localhost:3000"; // Local dev and Docker (internal calls) }; const lambdas = ["stripeRouter", "emailRouter"]; diff --git a/apps/private-location/.dockerignore b/apps/private-location/.dockerignore new file mode 100644 index 000000000..067cbb8dd --- /dev/null +++ b/apps/private-location/.dockerignore @@ -0,0 +1,3 @@ +# This file is generated by Dofigen v2.5.1 +# See https://github.com/lenra-io/dofigen + diff --git a/apps/private-location/Dockerfile b/apps/private-location/Dockerfile index 5e36f1d00..640633a4c 100644 --- a/apps/private-location/Dockerfile +++ b/apps/private-location/Dockerfile @@ -1,32 +1,65 @@ -FROM --platform=$BUILDPLATFORM golang:1.25-alpine as builder +# syntax=docker/dockerfile:1.11 +# This file is generated by Dofigen v2.5.1 +# See https://github.com/lenra-io/dofigen -ARG TARGETOS +# builder +FROM golang@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder ARG TARGETARCH - +ARG TARGETOS +LABEL \ + org.opencontainers.image.base.digest="sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb" \ + org.opencontainers.image.base.name="docker.io/golang:1.25-alpine" \ + org.opencontainers.image.stage="builder" +ENV \ + CGO_ENABLED="0" \ + TZ="UTC" WORKDIR /go/src/app +COPY \ + --link \ + "." "." +RUN < /// -import "./.next/types/routes.d.ts"; +import "./.next/dev/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/apps/status-page/next.config.ts b/apps/status-page/next.config.ts index bfdb44e67..4fb89f26e 100644 --- a/apps/status-page/next.config.ts +++ b/apps/status-page/next.config.ts @@ -1,8 +1,9 @@ -const { withSentryConfig } = require("@sentry/nextjs"); +import { withSentryConfig } from "@sentry/nextjs"; import type { NextConfig } from "next"; const nextConfig: NextConfig = { + ...(process.env.SELF_HOST === "true" && { output: "standalone" }), experimental: { authInterrupts: true, }, @@ -54,33 +55,24 @@ const nextConfig: NextConfig = { }, }; -module.exports = withSentryConfig( - nextConfig, - { - // For all available options, see: - // https://github.com/getsentry/sentry-webpack-plugin#options +// For detailed options, refer to the official documentation: +// - Webpack plugin options: https://github.com/getsentry/sentry-webpack-plugin#options +// - Next.js Sentry setup guide: https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/ +const sentryConfig = { + // Prevent log output unless running in a CI environment (helps reduce noise in logs) + silent: !process.env.CI, + org: "openstatus", + project: "openstatus", + authToken: process.env.SENTRY_AUTH_TOKEN, - // Only print logs for uploading source maps in CI - // Set to `true` to suppress logs - silent: !process.env.CI, + // Upload a larger set of source maps for improved stack trace accuracy (increases build time) + widenClientFileUpload: true, - org: "openstatus", - project: "openstatus", - }, - { - // For all available options, see: - // https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/ - - // Pass the auth token - authToken: process.env.SENTRY_AUTH_TOKEN, - - // Upload a larger set of source maps for prettier stack traces (increases build time) - widenClientFileUpload: true, + // If set to true, transpiles Sentry SDK to be compatible with IE11 (increases bundle size) + transpileClientSDK: false, - // Transpiles SDK to be compatible with IE11 (increases bundle size) - transpileClientSDK: false, + // Tree-shake Sentry logger statements to reduce bundle size + disableLogger: true, +}; - // Automatically tree-shake Sentry logger statements to reduce bundle size - disableLogger: true, - }, -); +export default withSentryConfig(nextConfig, sentryConfig); diff --git a/apps/status-page/src/app/(status-page)/[domain]/(public)/page.tsx b/apps/status-page/src/app/(status-page)/[domain]/(public)/page.tsx index c1cbd3ee3..edb30978e 100644 --- a/apps/status-page/src/app/(status-page)/[domain]/(public)/page.tsx +++ b/apps/status-page/src/app/(status-page)/[domain]/(public)/page.tsx @@ -28,7 +28,7 @@ import { Separator } from "@/components/ui/separator"; import { useTRPC } from "@/lib/trpc/client"; import { cn } from "@/lib/utils"; import { useQuery } from "@tanstack/react-query"; -import { useParams } from "next/navigation"; +import { notFound, useParams } from "next/navigation"; import { useMemo } from "react"; export default function Page() { @@ -38,12 +38,17 @@ export default function Page() { // NOTE: we cannot use `cardType` and `barType` here because of queryKey changes // It wouldn't match the server prefetch keys and we would have to refetch the page here - const { data: pageInitial } = useQuery( + const { data: pageInitial, error } = useQuery( trpc.statusPage.get.queryOptions({ slug: domain, }), ); + // Handle case where page doesn't exist or query fails + if (error || (!pageInitial && domain)) { + notFound(); + } + const hasCustomConfig = pageInitial?.configuration ? pageInitial.configuration.type !== barType || pageInitial.configuration.value !== cardType diff --git a/apps/status-page/src/components/nav/footer.tsx b/apps/status-page/src/components/nav/footer.tsx index 4c27d06db..1269c7846 100644 --- a/apps/status-page/src/components/nav/footer.tsx +++ b/apps/status-page/src/components/nav/footer.tsx @@ -14,9 +14,9 @@ export function Footer(props: React.ComponentProps<"footer">) { const { domain } = useParams<{ domain: string }>(); const [isMounted, setIsMounted] = useState(false); const trpc = useTRPC(); - const { data: page, dataUpdatedAt } = useQuery( - trpc.statusPage.get.queryOptions({ slug: domain }), - ); + const { data: page, dataUpdatedAt } = useQuery({ + ...trpc.statusPage.get.queryOptions({ slug: domain }), + }); const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone; useEffect(() => { diff --git a/apps/status-page/src/components/nav/header.tsx b/apps/status-page/src/components/nav/header.tsx index e7e343c51..7be819b6c 100644 --- a/apps/status-page/src/components/nav/header.tsx +++ b/apps/status-page/src/components/nav/header.tsx @@ -53,9 +53,9 @@ function useNav() { export function Header(props: React.ComponentProps<"header">) { const trpc = useTRPC(); const { domain } = useParams<{ domain: string }>(); - const { data: page } = useQuery( - trpc.statusPage.get.queryOptions({ slug: domain }), - ); + const { data: page } = useQuery({ + ...trpc.statusPage.get.queryOptions({ slug: domain }), + }); const sendPageSubscriptionMutation = useMutation( trpc.emailRouter.sendPageSubscription.mutationOptions({}), diff --git a/apps/status-page/src/hooks/use-pathname-prefix.ts b/apps/status-page/src/hooks/use-pathname-prefix.ts index 4a52b7a59..065fcd03b 100644 --- a/apps/status-page/src/hooks/use-pathname-prefix.ts +++ b/apps/status-page/src/hooks/use-pathname-prefix.ts @@ -8,9 +8,9 @@ import { useEffect, useState } from "react"; export function usePathnamePrefix() { const trpc = useTRPC(); const { domain } = useParams<{ domain: string }>(); - const { data: page } = useQuery( - trpc.statusPage.get.queryOptions({ slug: domain }), - ); + const { data: page } = useQuery({ + ...trpc.statusPage.get.queryOptions({ slug: domain }), + }); const [prefix, setPrefix] = useState(""); useEffect(() => { @@ -27,7 +27,7 @@ export function usePathnamePrefix() { ) { setPrefix(""); } else { - setPrefix(pathnames[1]); + setPrefix(pathnames[1] || ""); } } }, [page?.customDomain]); diff --git a/apps/status-page/src/lib/trpc/server.tsx b/apps/status-page/src/lib/trpc/server.tsx index 33cb12c65..44769136a 100644 --- a/apps/status-page/src/lib/trpc/server.tsx +++ b/apps/status-page/src/lib/trpc/server.tsx @@ -31,7 +31,7 @@ export const trpc = createTRPCOptionsProxy({ headers: { "x-trpc-source": "server", }, - async fetch(url, options) { + fetch: async (url, options) => { const cookieStore = await cookies(); return fetch(url, { ...options, diff --git a/apps/status-page/src/lib/trpc/shared.ts b/apps/status-page/src/lib/trpc/shared.ts index dfa3ca8f1..133fa3954 100644 --- a/apps/status-page/src/lib/trpc/shared.ts +++ b/apps/status-page/src/lib/trpc/shared.ts @@ -6,10 +6,9 @@ import superjson from "superjson"; const getBaseUrl = () => { if (typeof window !== "undefined") return ""; - const vc = process.env.VERCEL_URL; - if (vc) return `https://${vc}`; - // if (vc) return "https://app.openstatus.dev"; - return "http://localhost:3000"; + // Note: status-page has its own tRPC API routes + if (process.env.VERCEL_URL) return `https://${process.env.VERCEL_URL}`; // Vercel + return "http://localhost:3000"; // Local dev and Docker (internal calls) }; const lambdas = ["stripeRouter", "emailRouter"]; diff --git a/apps/status-page/src/proxy.ts b/apps/status-page/src/proxy.ts index 9a97e051c..d43343ddc 100644 --- a/apps/status-page/src/proxy.ts +++ b/apps/status-page/src/proxy.ts @@ -184,6 +184,14 @@ export const getValidSubdomain = (host?: string | null) => { // biome-ignore lint: to fix later host = window.location.host; } + + // Exclude localhost and IP addresses from being treated as subdomains + if ( + host?.match(/^(localhost|127\\.0\\.0\\.1|::1|\\d+\\.\\d+\\.\\d+\\.\\d+)/) + ) { + return null; + } + // we should improve here for custom vercel deploy page if (host?.includes(".") && !host.includes(".vercel.app")) { const candidate = host.split(".")[0]; diff --git a/apps/web/.env.example b/apps/web/.env.example index e137ee2d4..67f3311f5 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -50,7 +50,8 @@ GCP_CLIENT_EMAIL= GCP_PRIVATE_KEY= CRON_SECRET= -EXTERNAL_API_URL= +EXTERNAL_API_URL=http://server:3000 +VERCEL_URL=https://app.openstatus.dev PLAYGROUND_UNKEY_API_KEY= diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts index e8c20480f..eabf4f0e7 100644 --- a/apps/web/next.config.ts +++ b/apps/web/next.config.ts @@ -1,4 +1,4 @@ -const { withSentryConfig } = require("@sentry/nextjs"); +import { withSentryConfig } from "@sentry/nextjs"; import type { NextConfig } from "next"; // REMINDER: avoid Clickjacking attacks by setting the X-Frame-Options header @@ -221,33 +221,24 @@ const nextConfig: NextConfig = { }, }; -module.exports = withSentryConfig( - nextConfig, - { - // For all available options, see: - // https://github.com/getsentry/sentry-webpack-plugin#options - - // Only print logs for uploading source maps in CI - // Set to `true` to suppress logs - silent: !process.env.CI, +// For detailed options, refer to the official documentation: +// - Webpack plugin options: https://github.com/getsentry/sentry-webpack-plugin#options +// - Next.js Sentry setup guide: https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/ +const sentryConfig = { + // Prevent log output unless running in a CI environment (helps reduce noise in logs) + silent: !process.env.CI, + org: "openstatus", + project: "openstatus", + authToken: process.env.SENTRY_AUTH_TOKEN, - org: "openstatus", - project: "openstatus", - }, - { - // For all available options, see: - // https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/ + // Upload a larger set of source maps for improved stack trace accuracy (increases build time) + widenClientFileUpload: true, - // Pass the auth token - authToken: process.env.SENTRY_AUTH_TOKEN, + // If set to true, transpiles Sentry SDK to be compatible with IE11 (increases bundle size) + transpileClientSDK: false, - // Upload a larger set of source maps for prettier stack traces (increases build time) - widenClientFileUpload: true, - - // Transpiles SDK to be compatible with IE11 (increases bundle size) - transpileClientSDK: false, + // Tree-shake Sentry logger statements to reduce bundle size + disableLogger: true, +}; - // Automatically tree-shake Sentry logger statements to reduce bundle size - disableLogger: true, - }, -); +export default withSentryConfig(nextConfig, sentryConfig); diff --git a/apps/web/src/trpc/shared.ts b/apps/web/src/trpc/shared.ts index 9d9b3a03c..921106ff3 100644 --- a/apps/web/src/trpc/shared.ts +++ b/apps/web/src/trpc/shared.ts @@ -6,9 +6,8 @@ import superjson from "superjson"; const getBaseUrl = () => { if (typeof window !== "undefined") return ""; - const vc = process.env.VERCEL_URL; - if (vc) return `https://${vc}`; - return "http://localhost:3000"; + if (process.env.VERCEL_URL) return `https://${process.env.VERCEL_URL}`; // Vercel + return "http://localhost:3000"; // Local dev }; const lambdas = ["stripeRouter", "emailRouter"]; diff --git a/apps/workflows/Dockerfile b/apps/workflows/Dockerfile index baff4e39d..3da800b74 100644 --- a/apps/workflows/Dockerfile +++ b/apps/workflows/Dockerfile @@ -3,17 +3,19 @@ # See https://github.com/lenra-io/dofigen # ca-certs -FROM debian@sha256:52927eff8153b563244f98cdc802ba97918afcdf67f9e4867cbf1f7afb3d147b AS ca-certs +FROM debian@sha256:530a3348fc4b5734ffe1a137ddbcee6850154285251b53c3425c386ea8fac77b AS ca-certs LABEL \ - org.opencontainers.image.base.digest="sha256:52927eff8153b563244f98cdc802ba97918afcdf67f9e4867cbf1f7afb3d147b" \ - org.opencontainers.image.base.name="docker.io/debian:bullseye-slim" -RUN apt update && apt install -y ca-certificates && update-ca-certificates + org.opencontainers.image.base.digest="sha256:530a3348fc4b5734ffe1a137ddbcee6850154285251b53c3425c386ea8fac77b" \ + org.opencontainers.image.base.name="docker.io/debian:bullseye-slim" \ + org.opencontainers.image.stage="ca-certs" +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && update-ca-certificates && rm -rf /var/lib/apt/lists/* # docker FROM oven/bun@sha256:fbf8e67e9d3b806c86be7a2f2e9bae801f2d9212a21db4dcf8cc9889f5a3c9c4 AS docker LABEL \ org.opencontainers.image.base.digest="sha256:fbf8e67e9d3b806c86be7a2f2e9bae801f2d9212a21db4dcf8cc9889f5a3c9c4" \ - org.opencontainers.image.base.name="docker.io/oven/bun:1.3.3" + org.opencontainers.image.base.name="docker.io/oven/bun:1.3.3" \ + org.opencontainers.image.stage="docker" WORKDIR /app/apps/workflows COPY \ --link \ @@ -24,7 +26,8 @@ RUN bun run src/build-docker.ts FROM oven/bun@sha256:fbf8e67e9d3b806c86be7a2f2e9bae801f2d9212a21db4dcf8cc9889f5a3c9c4 AS install LABEL \ org.opencontainers.image.base.digest="sha256:fbf8e67e9d3b806c86be7a2f2e9bae801f2d9212a21db4dcf8cc9889f5a3c9c4" \ - org.opencontainers.image.base.name="docker.io/oven/bun:1.3.3" + org.opencontainers.image.base.name="docker.io/oven/bun:1.3.3" \ + org.opencontainers.image.stage="install" WORKDIR /app/ RUN \ --mount=type=bind,target=bunfig.toml,source=bunfig.toml \ @@ -48,13 +51,14 @@ RUN \ --mount=type=bind,target=packages/upstash/package.json,source=packages/upstash/package.json \ --mount=type=bind,target=packages/theme-store/package.json,source=packages/theme-store/package.json \ --mount=type=cache,target=/root/.bun/install/cache,sharing=locked \ - bun install --production --frozen-lockfile --verbose + bun install --production --frozen-lockfile --verbose # build FROM oven/bun@sha256:fbf8e67e9d3b806c86be7a2f2e9bae801f2d9212a21db4dcf8cc9889f5a3c9c4 AS build LABEL \ org.opencontainers.image.base.digest="sha256:fbf8e67e9d3b806c86be7a2f2e9bae801f2d9212a21db4dcf8cc9889f5a3c9c4" \ - org.opencontainers.image.base.name="docker.io/oven/bun:1.3.3" + org.opencontainers.image.base.name="docker.io/oven/bun:1.3.3" \ + org.opencontainers.image.stage="build" ENV NODE_ENV="production" WORKDIR /app/apps/workflows COPY \ @@ -70,7 +74,8 @@ RUN bun build --compile --target bun --sourcemap --format=cjs src/index.ts --ou FROM oven/bun@sha256:fbf8e67e9d3b806c86be7a2f2e9bae801f2d9212a21db4dcf8cc9889f5a3c9c4 AS libsql LABEL \ org.opencontainers.image.base.digest="sha256:fbf8e67e9d3b806c86be7a2f2e9bae801f2d9212a21db4dcf8cc9889f5a3c9c4" \ - org.opencontainers.image.base.name="docker.io/oven/bun:1.3.3" + org.opencontainers.image.base.name="docker.io/oven/bun:1.3.3" \ + org.opencontainers.image.stage="libsql" WORKDIR /app/ COPY \ --from=docker \ @@ -79,11 +84,16 @@ COPY \ RUN bun install # runtime -FROM debian@sha256:52927eff8153b563244f98cdc802ba97918afcdf67f9e4867cbf1f7afb3d147b AS runtime +FROM debian@sha256:530a3348fc4b5734ffe1a137ddbcee6850154285251b53c3425c386ea8fac77b AS runtime LABEL \ io.dofigen.version="2.5.1" \ - org.opencontainers.image.base.digest="sha256:52927eff8153b563244f98cdc802ba97918afcdf67f9e4867cbf1f7afb3d147b" \ - org.opencontainers.image.base.name="docker.io/debian:bullseye-slim" + org.opencontainers.image.authors="OpenStatus Team" \ + org.opencontainers.image.base.digest="sha256:530a3348fc4b5734ffe1a137ddbcee6850154285251b53c3425c386ea8fac77b" \ + org.opencontainers.image.base.name="docker.io/debian:bullseye-slim" \ + org.opencontainers.image.description="Background job processing and probe scheduling for OpenStatus" \ + org.opencontainers.image.source="https://github.com/openstatusHQ/openstatus" \ + org.opencontainers.image.title="OpenStatus Workflows" \ + org.opencontainers.image.vendor="OpenStatus" WORKDIR /app/ COPY \ --from=build \ @@ -91,6 +101,11 @@ COPY \ --chmod=555 \ --link \ "/app/apps/workflows/app" "/app/apps/workflows/" +COPY \ + --from=build \ + --chown=1000:1000 \ + --link \ + "/app/data" "/app/data" COPY \ --from=libsql \ --chown=1000:1000 \ @@ -106,6 +121,18 @@ COPY \ --chown=1000:1000 \ --link \ "/etc/ssl/certs/ca-certificates.crt" "/etc/ssl/certs/" +USER 0:0 +RUN <new(PeerAddr=>\"127.0.0.1:8080\",Timeout=>1) ? 0 : 1);'"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + restart: unless-stopped + + tinybird-local: + container_name: openstatus-tinybird + image: tinybirdco/tinybird-local:latest + platform: linux/amd64 + networks: + - openstatus + ports: + - "7181:7181" + environment: + - COMPATIBILITY_MODE=1 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:7181/"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + restart: unless-stopped + + # Internal Services + workflows: + container_name: openstatus-workflows build: - context: . - dockerfile: apps/workflows/Dockerfile + context: . + dockerfile: apps/workflows/Dockerfile + image: openstatus/workflows:latest + networks: + - openstatus + ports: + - "3000:3000" + env_file: + - .env.docker environment: - DATABASE_URL=http://libsql:8080 - - DATABASE_AUTH_TOKEN="basic:token" - - RESEND_API_KEY=test - - UPSTASH_REDIS_REST_URL=test - - UPSTASH_REDIS_REST_TOKEN=test - - NODE_ENV=production - - GCP_PROJECT_ID=test - extra_hosts: - - "host.docker.internal:host-gateway" + - PORT=3000 + depends_on: + libsql: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:3000/ping || exit 1"] + interval: 15s + timeout: 10s + retries: 3 + start_period: 30s + restart: unless-stopped + server: + container_name: openstatus-server + build: + context: . + dockerfile: apps/server/Dockerfile + image: openstatus/server:latest + networks: + - openstatus ports: - - 3000:3000 - volumes: - - type: bind - source: ./data - target: /app/data - image: workflows-test - command: . - libsql: - image: ghcr.io/tursodatabase/libsql-server:latest + - "3001:3000" + env_file: + - .env.docker + environment: + - DATABASE_URL=http://libsql:8080 + - PORT=3000 + depends_on: + libsql: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:3000/ping || exit 1"] + interval: 15s + timeout: 10s + retries: 3 + start_period: 30s + restart: unless-stopped + + private-location: + container_name: openstatus-private-location + build: + context: apps/private-location + dockerfile: Dockerfile + image: openstatus/private-location:latest + networks: + - openstatus + ports: + - "8081:8080" + env_file: + - .env.docker + environment: + - DB_URL=http://libsql:8080 + - TINYBIRD_URL=http://tinybird-local:7181 + - GIN_MODE=release + - PORT=8080 + depends_on: + server: + condition: service_started + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/health"] + interval: 15s + timeout: 10s + retries: 3 + start_period: 30s + restart: unless-stopped + + dashboard: + container_name: openstatus-dashboard + build: + context: . + dockerfile: apps/dashboard/Dockerfile + image: openstatus/dashboard:latest + networks: + - openstatus ports: - - 8080:8080 - - 5001:5001 - api: + - "3002:3000" + env_file: + - .env.docker + environment: + - DATABASE_URL=http://libsql:8080 + - PORT=3000 + - HOSTNAME=0.0.0.0 + - AUTH_TRUST_HOST=true + depends_on: + libsql: + condition: service_healthy + server: + condition: service_started + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:3000/ || exit 1"] + interval: 15s + timeout: 10s + retries: 3 + start_period: 45s + restart: unless-stopped + + status-page: + container_name: openstatus-status-page build: - context: . - dockerfile: apps/server/Dockerfile + context: . + dockerfile: apps/status-page/Dockerfile + image: openstatus/status-page:latest + networks: + - openstatus ports: - - 3001:3000 - image: server + - "3003:3000" + env_file: + - .env.docker environment: - - DATABASE_URL=http://libsql-1:8080 - - DATABASE_AUTH_TOKEN="basic:token" - - UPSTASH_REDIS_REST_URL=test - - UPSTASH_REDIS_REST_TOKEN=test - - NODE_ENV=production - - NEXT_PUBLIC_OPENPANEL_CLIENT_ID=test - - OPENPANEL_CLIENT_SECRET=test - - RESEND_API_KEY=test - command: . - # dashboard: - # build: - # context: . - # dockerfile: apps/dashboard/Dockerfile - # ports: - # - 3002:3000 - # image: dashboard - # environment: - # - API_URL=http://host.docker.internal:3001 - # - NEXT_PUBLIC_OPENPANEL_CLIENT_ID=test - # - OPENPANEL_CLIENT_SECRET=test - # - NODE_ENV=production - # command: . + - DATABASE_URL=http://libsql:8080 + - PORT=3000 + - HOSTNAME=0.0.0.0 + - AUTH_TRUST_HOST=true + depends_on: + libsql: + condition: service_healthy + server: + condition: service_started + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:3000/ || exit 1"] + interval: 15s + timeout: 10s + retries: 3 + start_period: 45s + restart: unless-stopped diff --git a/packages/tinybird/_legacy/aggregated_monitor_day_mv.pipe b/packages/tinybird/_legacy/aggregated_monitor_day_mv.pipe deleted file mode 100644 index da595cbff..000000000 --- a/packages/tinybird/_legacy/aggregated_monitor_day_mv.pipe +++ /dev/null @@ -1,25 +0,0 @@ -# REMINDER: legacy - -VERSION 0 - -NODE aggregated_monitor_day_0 -SQL > - - SELECT - toUnixTimestamp64Milli(toDateTime64(DATE(cronTimestamp / 1000), 3)) AS day, - monitorId, - avgState(latency) AS avgLatency, - countState() AS count, - countState(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok - FROM ping_response__v4 - WHERE (day IS NOT NULL) AND (day != 0) - GROUP BY - day, - monitorId - ORDER BY day DESC - -TYPE materialized -DATASOURCE aggregated_monitor_per_day_0_mv -ENGINE "AggregatingMergeTree" -ENGINE_SORTING_KEY "day, monitorId" - diff --git a/packages/tinybird/_legacy/aggregated_monitor_per_day_mv.datasource b/packages/tinybird/_legacy/aggregated_monitor_per_day_mv.datasource deleted file mode 100644 index f3a05b093..000000000 --- a/packages/tinybird/_legacy/aggregated_monitor_per_day_mv.datasource +++ /dev/null @@ -1,14 +0,0 @@ -# REMINDER: legacy - -# Data Source created from Pipe 'aggregated_monitor_day_mv' -VERSION 0 - -SCHEMA > - `day` Int64, - `monitorId` String, - `avgLatency` AggregateFunction(avg, Int16), - `count` AggregateFunction(count), - `ok` AggregateFunction(count, Nullable(UInt8)) - -ENGINE "AggregatingMergeTree" -ENGINE_SORTING_KEY "day, monitorId" diff --git a/packages/tinybird/_migration/tb_backfill_populate.pipe b/packages/tinybird/_migration/tb_backfill_populate.pipe deleted file mode 100644 index 848b8974a..000000000 --- a/packages/tinybird/_migration/tb_backfill_populate.pipe +++ /dev/null @@ -1,21 +0,0 @@ -NODE mat_node -SQL > - - SELECT - latency, - monitorId, - toLowCardinality(region) region, - statusCode, - if(statusCode >= 200 AND statusCode < 300, toInt8(0), toInt8(1)) AS error, - timestamp, - url, - workspaceId, - cronTimestamp, - message, - timing, - headers, - FROM ping_response__v7 - WHERE fromUnixTimestamp64Milli(cronTimestamp) <= '2024-03-29 13:16:00.000' - -TYPE materialized -DATASOURCE ping_response__v8 \ No newline at end of file diff --git a/packages/tinybird/_migration/tb_datasource_union.pipe b/packages/tinybird/_migration/tb_datasource_union.pipe deleted file mode 100644 index 2b60019c7..000000000 --- a/packages/tinybird/_migration/tb_datasource_union.pipe +++ /dev/null @@ -1,6 +0,0 @@ -NODE union_all -SQL > - - SELECT count() FROM ping_response__v8 - UNION ALL - SELECT count() FROM ping_response__v7 diff --git a/packages/tinybird/_migration/tb_materialize_until_change_ingest.pipe b/packages/tinybird/_migration/tb_materialize_until_change_ingest.pipe deleted file mode 100644 index bff97f1b1..000000000 --- a/packages/tinybird/_migration/tb_materialize_until_change_ingest.pipe +++ /dev/null @@ -1,21 +0,0 @@ -NODE mat_node -SQL > - - SELECT - latency, - monitorId, - toLowCardinality(region) region, - statusCode, - if(statusCode >= 200 AND statusCode < 300, toInt8(0), toInt8(1)) AS error, - timestamp, - url, - workspaceId, - cronTimestamp, - message, - timing, - headers, - FROM ping_response__v7 - WHERE fromUnixTimestamp64Milli(cronTimestamp) > '2023-03-29 13:16:00.000' - -TYPE materialized -DATASOURCE ping_response__v8 \ No newline at end of file diff --git a/packages/tinybird/datasources/audit_log.datasource b/packages/tinybird/datasources/audit_log__v0.datasource similarity index 100% rename from packages/tinybird/datasources/audit_log.datasource rename to packages/tinybird/datasources/audit_log__v0.datasource diff --git a/packages/tinybird/datasources/mv__http_14d__v0.datasource b/packages/tinybird/datasources/mv__http_14d__v0.datasource new file mode 100644 index 000000000..5195321a6 --- /dev/null +++ b/packages/tinybird/datasources/mv__http_14d__v0.datasource @@ -0,0 +1,19 @@ +VERSION 0 +# Data Source created from Pipe 'aggregate__http_14d__v0' + +SCHEMA > + `time` DateTime, + `latency` Int64, + `error` Int8, + `region` LowCardinality(String), + `trigger` Nullable(String), + `statusCode` Nullable(Int16), + `timestamp` Int64, + `cronTimestamp` Int64, + `monitorId` String, + `workspaceId` String + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(time)" +ENGINE_SORTING_KEY "monitorId, time" +ENGINE_TTL "time + toIntervalDay(14)" diff --git a/packages/tinybird/datasources/mv__http_1d.datasource b/packages/tinybird/datasources/mv__http_1d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__http_1d.datasource rename to packages/tinybird/datasources/mv__http_1d__v0.datasource diff --git a/packages/tinybird/datasources/mv__http_30d.datasource b/packages/tinybird/datasources/mv__http_30d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__http_30d.datasource rename to packages/tinybird/datasources/mv__http_30d__v0.datasource diff --git a/packages/tinybird/datasources/mv__http_7d.datasource b/packages/tinybird/datasources/mv__http_7d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__http_7d.datasource rename to packages/tinybird/datasources/mv__http_7d__v0.datasource diff --git a/packages/tinybird/datasources/mv__http_full_14d__v0.datasource b/packages/tinybird/datasources/mv__http_full_14d__v0.datasource index 3b3df7108..95a905e1c 100644 --- a/packages/tinybird/datasources/mv__http_full_14d__v0.datasource +++ b/packages/tinybird/datasources/mv__http_full_14d__v0.datasource @@ -18,7 +18,9 @@ SCHEMA > `body` Nullable(String), `trigger` Nullable(String), `id` Nullable(String), - `requestStatus` Nullable(String) + `requestStatus` Nullable(String), + `method` Nullable(String) + ENGINE "MergeTree" ENGINE_PARTITION_KEY "toYYYYMM(time)" diff --git a/packages/tinybird/datasources/mv__http_full_30d.datasource b/packages/tinybird/datasources/mv__http_full_30d__v0.datasource similarity index 82% rename from packages/tinybird/datasources/mv__http_full_30d.datasource rename to packages/tinybird/datasources/mv__http_full_30d__v0.datasource index 8a86b69b0..fbf149752 100644 --- a/packages/tinybird/datasources/mv__http_full_30d.datasource +++ b/packages/tinybird/datasources/mv__http_full_30d__v0.datasource @@ -1,4 +1,3 @@ -VERSION 0 # Data Source created from Pipe 'aggregate__http_full_30d__v0' SCHEMA > @@ -17,7 +16,11 @@ SCHEMA > `headers` Nullable(String), `assertions` Nullable(String), `body` Nullable(String), - `trigger` Nullable(String) + `trigger` Nullable(String), + `id` Nullable(String), + `requestStatus` Nullable(String), + `method` Nullable(String) + ENGINE "MergeTree" ENGINE_PARTITION_KEY "toYYYYMM(time)" diff --git a/packages/tinybird/datasources/mv__http_status_14d__v0.datasource b/packages/tinybird/datasources/mv__http_status_14d__v0.datasource new file mode 100644 index 000000000..6a7316f76 --- /dev/null +++ b/packages/tinybird/datasources/mv__http_status_14d__v0.datasource @@ -0,0 +1,14 @@ +# Data Source created from Pipe 'aggregate__http_status_14d__v0' + +SCHEMA > + `time` DateTime('UTC'), + `monitorId` String, + `total` AggregateFunction(count), + `success` AggregateFunction(count, Nullable(UInt8)), + `degraded` AggregateFunction(count, Nullable(UInt8)), + `error` AggregateFunction(count, Nullable(UInt8)) + +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(time)" +ENGINE_SORTING_KEY "monitorId, time" +ENGINE_TTL "time + toIntervalDay(14)" diff --git a/packages/tinybird/datasources/mv__http_status_45d.datasource b/packages/tinybird/datasources/mv__http_status_45d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__http_status_45d.datasource rename to packages/tinybird/datasources/mv__http_status_45d__v0.datasource diff --git a/packages/tinybird/datasources/mv__http_status_7d.datasource b/packages/tinybird/datasources/mv__http_status_7d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__http_status_7d.datasource rename to packages/tinybird/datasources/mv__http_status_7d__v0.datasource diff --git a/packages/tinybird/datasources/mv__http_timing_phases_14d__v1.datasource b/packages/tinybird/datasources/mv__http_timing_phases_14d__v1.datasource new file mode 100644 index 000000000..b767183d9 --- /dev/null +++ b/packages/tinybird/datasources/mv__http_timing_phases_14d__v1.datasource @@ -0,0 +1,21 @@ +# Data Source created from Pipe 'aggregate__http_timing_phases_14d__v1' + +SCHEMA > + `time` DateTime, + `latency` Int64, + `region` LowCardinality(String), + `trigger` Nullable(String), + `statusCode` Nullable(Int16), + `monitorId` String, + `workspaceId` String, + `requestStatus` Nullable(String), + `dns` Nullable(Int64), + `connect` Nullable(Int64), + `tls` Nullable(Int64), + `firstByte` Nullable(Int64), + `transfer` Nullable(Int64) + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(time)" +ENGINE_SORTING_KEY "monitorId, time" +ENGINE_TTL "time + toIntervalDay(14)" diff --git a/packages/tinybird/datasources/mv__http_uptime_30d.datasource b/packages/tinybird/datasources/mv__http_uptime_30d__v1.datasource similarity index 97% rename from packages/tinybird/datasources/mv__http_uptime_30d.datasource rename to packages/tinybird/datasources/mv__http_uptime_30d__v1.datasource index 617982619..1e1a0a6d8 100644 --- a/packages/tinybird/datasources/mv__http_uptime_30d.datasource +++ b/packages/tinybird/datasources/mv__http_uptime_30d__v1.datasource @@ -1,5 +1,4 @@ # Data Source created from Pipe 'aggregate__http_uptime_30d__v1' -VERSION 1 SCHEMA > `time` DateTime, diff --git a/packages/tinybird/datasources/mv__tcp_14d.datasource b/packages/tinybird/datasources/mv__tcp_14d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__tcp_14d.datasource rename to packages/tinybird/datasources/mv__tcp_14d__v0.datasource diff --git a/packages/tinybird/datasources/mv__tcp_1d.datasource b/packages/tinybird/datasources/mv__tcp_1d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__tcp_1d.datasource rename to packages/tinybird/datasources/mv__tcp_1d__v0.datasource diff --git a/packages/tinybird/datasources/mv__tcp_30d.datasource b/packages/tinybird/datasources/mv__tcp_30d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__tcp_30d.datasource rename to packages/tinybird/datasources/mv__tcp_30d__v0.datasource diff --git a/packages/tinybird/datasources/mv__tcp_7d.datasource b/packages/tinybird/datasources/mv__tcp_7d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__tcp_7d.datasource rename to packages/tinybird/datasources/mv__tcp_7d__v0.datasource diff --git a/packages/tinybird/datasources/mv__tcp_full_30d.datasource b/packages/tinybird/datasources/mv__tcp_full_30d__v0.datasource similarity index 83% rename from packages/tinybird/datasources/mv__tcp_full_30d.datasource rename to packages/tinybird/datasources/mv__tcp_full_30d__v0.datasource index 86e7f9c46..9bcd02b23 100644 --- a/packages/tinybird/datasources/mv__tcp_full_30d.datasource +++ b/packages/tinybird/datasources/mv__tcp_full_30d__v0.datasource @@ -1,4 +1,3 @@ -VERSION 0 # Data Source created from Pipe 'aggregate__tcp_full_30d__v0' SCHEMA > @@ -13,7 +12,9 @@ SCHEMA > `errorMessage` Nullable(String), `error` Int16, `trigger` Nullable(String), - `uri` Nullable(String) + `uri` Nullable(String), + `id` Nullable(String), + `requestStatus` Nullable(String) ENGINE "MergeTree" ENGINE_PARTITION_KEY "toYYYYMM(time)" diff --git a/packages/tinybird/datasources/mv__tcp_status_45d.datasource b/packages/tinybird/datasources/mv__tcp_status_45d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__tcp_status_45d.datasource rename to packages/tinybird/datasources/mv__tcp_status_45d__v0.datasource diff --git a/packages/tinybird/datasources/mv__tcp_status_7d.datasource b/packages/tinybird/datasources/mv__tcp_status_7d__v0.datasource similarity index 100% rename from packages/tinybird/datasources/mv__tcp_status_7d.datasource rename to packages/tinybird/datasources/mv__tcp_status_7d__v0.datasource diff --git a/packages/tinybird/datasources/ping_response.datasource b/packages/tinybird/datasources/ping_response.datasource deleted file mode 100644 index 750dcad0c..000000000 --- a/packages/tinybird/datasources/ping_response.datasource +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 8 - -SCHEMA > - `latency` Int64 `json:$.latency`, - `monitorId` String `json:$.monitorId`, - `region` LowCardinality(String) `json:$.region`, - `statusCode` Nullable(Int16) `json:$.statusCode`, - `error` Int8 `json:$.error`, - `timestamp` Int64 `json:$.timestamp`, - `url` String `json:$.url`, - `workspaceId` String `json:$.workspaceId`, - `cronTimestamp` Int64 `json:$.cronTimestamp`, - `message` Nullable(String) `json:$.message`, - `timing` Nullable(String) `json:$.timing`, - `headers` Nullable(String) `json:$.headers`, - `assertions` Nullable(String) `json:$.assertions`, - `body` Nullable(String) `json:$.body`, - `trigger` Nullable(String) `json:$.trigger`, - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "monitorId, cronTimestamp" -ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(cronTimestamp))" \ No newline at end of file diff --git a/packages/tinybird/_migration/ping_response.datasource b/packages/tinybird/datasources/ping_response__v8.datasource similarity index 63% rename from packages/tinybird/_migration/ping_response.datasource rename to packages/tinybird/datasources/ping_response__v8.datasource index 10b3ffa33..10144240a 100644 --- a/packages/tinybird/_migration/ping_response.datasource +++ b/packages/tinybird/datasources/ping_response__v8.datasource @@ -1,4 +1,3 @@ -VERSION 8 SCHEMA > `latency` Int64 `json:$.latency`, @@ -13,8 +12,13 @@ SCHEMA > `message` Nullable(String) `json:$.message`, `timing` Nullable(String) `json:$.timing`, `headers` Nullable(String) `json:$.headers`, - `assertions` Nullable(String) `json:$.assertions` + `assertions` Nullable(String) `json:$.assertions`, + `body` Nullable(String) `json:$.body`, + `trigger` Nullable(String) `json:$.trigger`, + `id` Nullable(String) `json:$.id`, + `requestStatus` Nullable(String) `json:$.requestStatus`, + `method` String `json:$.method` ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(cronTimestamp))" ENGINE_SORTING_KEY "monitorId, cronTimestamp" -ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(cronTimestamp))" \ No newline at end of file diff --git a/packages/tinybird/datasources/tcp_response__v0.datasource b/packages/tinybird/datasources/tcp_response__v0.datasource new file mode 100644 index 000000000..ae4f7aa97 --- /dev/null +++ b/packages/tinybird/datasources/tcp_response__v0.datasource @@ -0,0 +1,19 @@ + +SCHEMA > + `monitorId` Int32 `json:$.monitorId`, + `region` String `json:$.region`, + `timestamp` Int64 `json:$.timestamp`, + `cronTimestamp` Int64 `json:$.timestamp`, + `timing` String `json:$.timing`, + `workspaceId` Int32 `json:$.workspaceId`, + `latency` Int64 `json:$.latency`, + `errorMessage` Nullable(String) `json:$.errorMessage`, + `error` Int16 `json:$.error`, + `trigger` Nullable(String) `json:$.trigger`, + `uri` Nullable(String) `json:$.uri`, + `id` Nullable(String) `json:$.id`, + `requestStatus` Nullable(String) `json:$.requestStatus` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(timestamp))" +ENGINE_SORTING_KEY "monitorId, workspaceId" diff --git a/packages/tinybird/datasources/web_vitals.datasource b/packages/tinybird/datasources/web_vitals.datasource deleted file mode 100644 index 403d40195..000000000 --- a/packages/tinybird/datasources/web_vitals.datasource +++ /dev/null @@ -1,25 +0,0 @@ -VERSION 0 - -SCHEMA > - `browser` String `json:$.browser`, - `city` String `json:$.city`, - `continent` String `json:$.continent`, - `country` String `json:$.country`, - `device` String `json:$.device`, - `dsn` String `json:$.dsn`, - `href` String `json:$.href`, - `id` String `json:$.id`, - `name` String `json:$.name`, - `os` String `json:$.os`, - `path` String `json:$.path`, - `rating` String `json:$.rating`, - `region_code` String `json:$.region_code`, - `screen` String `json:$.screen`, - `session_id` String `json:$.session_id`, - `speed` String `json:$.speed`, - `timezone` String `json:$.timezone`, - `value` Float32 `json:$.value`, - `timestamp` Nullable(DateTime) `json:$.timestamp` - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "session_id, speed, timezone" diff --git a/packages/tinybird/pipes/endpoint__audit_log.pipe b/packages/tinybird/endpoints/endpoint__audit_log.pipe similarity index 95% rename from packages/tinybird/pipes/endpoint__audit_log.pipe rename to packages/tinybird/endpoints/endpoint__audit_log.pipe index a5eed224e..484e795b3 100644 --- a/packages/tinybird/pipes/endpoint__audit_log.pipe +++ b/packages/tinybird/endpoints/endpoint__audit_log.pipe @@ -14,4 +14,4 @@ SQL > AND timestamp > toUnixTimestamp(now() - INTERVAL {{ Int64(interval, 30) }} day) * 1000 ORDER BY timestamp DESC - +TYPE ENDPOINT diff --git a/packages/tinybird/endpoints/endpoint__audit_log__v1.pipe b/packages/tinybird/endpoints/endpoint__audit_log__v1.pipe new file mode 100644 index 000000000..5ad399aca --- /dev/null +++ b/packages/tinybird/endpoints/endpoint__audit_log__v1.pipe @@ -0,0 +1,18 @@ +VERSION 1 + +TAGS endpoint + + +NODE endpoint +SQL > + + % + SELECT action, id, metadata, timestamp + FROM audit_log__v0 + WHERE + id = {{String(monitorId, 'monitor:1', required=True)}} + AND timestamp > toUnixTimestamp(now() - INTERVAL {{ Int64(interval, 30) }} day) * 1000 + ORDER BY timestamp DESC + +TYPE ENDPOINT + diff --git a/packages/tinybird/pipes/endpoint__dns_get_14d__v0.pipe b/packages/tinybird/endpoints/endpoint__dns_get_14d__v0.pipe similarity index 94% rename from packages/tinybird/pipes/endpoint__dns_get_14d__v0.pipe rename to packages/tinybird/endpoints/endpoint__dns_get_14d__v0.pipe index 2aac7511d..1edb0b5d8 100644 --- a/packages/tinybird/pipes/endpoint__dns_get_14d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_get_14d__v0.pipe @@ -12,3 +12,4 @@ SQL > ORDER BY timestamp DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__dns_list_14d__v0.pipe b/packages/tinybird/endpoints/endpoint__dns_list_14d__v0.pipe similarity index 96% rename from packages/tinybird/pipes/endpoint__dns_list_14d__v0.pipe rename to packages/tinybird/endpoints/endpoint__dns_list_14d__v0.pipe index 5c31bdf35..fce5897d7 100644 --- a/packages/tinybird/pipes/endpoint__dns_list_14d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_list_14d__v0.pipe @@ -13,3 +13,4 @@ SQL > ORDER BY timestamp DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_1d__v1.pipe b/packages/tinybird/endpoints/endpoint__dns_metrics_14d__v0.pipe similarity index 73% rename from packages/tinybird/pipes/endpoint__http_metrics_1d__v1.pipe rename to packages/tinybird/endpoints/endpoint__dns_metrics_14d__v0.pipe index 3e68df5d2..18385a356 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_1d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_metrics_14d__v0.pipe @@ -1,4 +1,4 @@ -TAGS "http" +TAGS "dns" NODE endpoint SQL > @@ -15,10 +15,10 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, max(cronTimestamp) AS lastTimestamp - FROM mv__http_1d__v1 + FROM dns_response__v0 WHERE - monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 1 DAY, 3) + monitorId = {{ String(monitorId, '7417', required=True) }} + AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 14 DAY, 3)) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} UNION ALL SELECT @@ -32,11 +32,12 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__http_7d__v1 + FROM dns_response__v0 WHERE - monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 2 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 1 DAY, 3) + monitorId = {{ String(monitorId, '7417', required=True) }} + AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 28 DAY, 3)) + AND timestamp < toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 14 DAY, 3)) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_7d__v1.pipe b/packages/tinybird/endpoints/endpoint__dns_metrics_1d__v0.pipe similarity index 73% rename from packages/tinybird/pipes/endpoint__tcp_metrics_7d__v1.pipe rename to packages/tinybird/endpoints/endpoint__dns_metrics_1d__v0.pipe index 862d38c7b..7db3011dc 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_7d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_metrics_1d__v0.pipe @@ -1,4 +1,4 @@ -TAGS "tcp" +TAGS "dns" NODE endpoint SQL > @@ -15,10 +15,10 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, max(cronTimestamp) AS lastTimestamp - FROM mv__tcp_7d__v1 + FROM dns_response__v0 WHERE - monitorId = {{ String(monitorId, '4433', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 7 DAY, 3) + monitorId = {{ String(monitorId, '7417', required=True) }} + AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 1 DAY, 3)) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} UNION ALL SELECT @@ -32,11 +32,12 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__tcp_14d__v1 + FROM dns_response__v0 WHERE - monitorId = {{ String(monitorId, '4433', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 7 DAY, 3) + monitorId = {{ String(monitorId, '7417', required=True) }} + AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 2 DAY, 3)) + AND timestamp < toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 1 DAY, 3)) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_14d__v1.pipe b/packages/tinybird/endpoints/endpoint__dns_metrics_7d__v0.pipe similarity index 73% rename from packages/tinybird/pipes/endpoint__http_metrics_14d__v1.pipe rename to packages/tinybird/endpoints/endpoint__dns_metrics_7d__v0.pipe index 56ac581dc..731df660b 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_14d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_metrics_7d__v0.pipe @@ -1,4 +1,4 @@ -TAGS "http" +TAGS "dns" NODE endpoint SQL > @@ -15,10 +15,10 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, max(cronTimestamp) AS lastTimestamp - FROM mv__http_14d__v1 + FROM dns_response__v0 WHERE - monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) + monitorId = {{ String(monitorId, '7417', required=True) }} + AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 7 DAY, 3)) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} UNION ALL SELECT @@ -32,11 +32,12 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__http_30d__v1 + FROM dns_response__v0 WHERE - monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 28 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 14 DAY, 3) + monitorId = {{ String(monitorId, '7417', required=True) }} + AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 14 DAY, 3)) + AND timestamp < toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 7 DAY, 3)) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_by_interval_14d.pipe b/packages/tinybird/endpoints/endpoint__dns_metrics_latency_1d_multi__v0.pipe similarity index 54% rename from packages/tinybird/pipes/endpoint__tcp_metrics_by_interval_14d.pipe rename to packages/tinybird/endpoints/endpoint__dns_metrics_latency_1d_multi__v0.pipe index 590acbc13..74968f620 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_by_interval_14d.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_metrics_latency_1d_multi__v0.pipe @@ -1,27 +1,26 @@ -VERSION 0 - -TAGS tcp +TAGS "dns" NODE endpoint SQL > % SELECT - region, toStartOfInterval( - toDateTime(cronTimestamp / 1000), - INTERVAL {{ Int64(interval, 30) }} MINUTE -- use 2880 (2d) in case you want the 1d summary for the regions + toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, + monitorId, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, round(quantile(0.75)(latency)) as p75Latency, round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__tcp_14d__v0 + FROM dns_response__v0 WHERE - monitorId = {{ String(monitorId, '1', required=True) }} - GROUP BY h, region - ORDER BY h DESC + monitorId IN {{ Array(monitorIds, 'String', '7417') }} + AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 1 DAY, 3)) + GROUP BY h, monitorId + ORDER BY h ASC +TYPE ENDPOINT diff --git a/packages/tinybird/endpoints/endpoint__dns_metrics_latency_7d__v0.pipe b/packages/tinybird/endpoints/endpoint__dns_metrics_latency_7d__v0.pipe new file mode 100644 index 000000000..d9e3e3db0 --- /dev/null +++ b/packages/tinybird/endpoints/endpoint__dns_metrics_latency_7d__v0.pipe @@ -0,0 +1,30 @@ +TAGS "dns" + +NODE endpoint +SQL > + + % + SELECT + toStartOfInterval( + toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE + ) as h, + toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, + round(quantile(0.50)(latency)) as p50Latency, + round(quantile(0.75)(latency)) as p75Latency, + round(quantile(0.90)(latency)) as p90Latency, + round(quantile(0.95)(latency)) as p95Latency, + round(quantile(0.99)(latency)) as p99Latency + FROM dns_response__v0 + WHERE + monitorId = {{ String(monitorId, '7417', required=True) }} + {% if defined(fromDate) %} + AND toDateTime(timestamp / 1000) >= parseDateTimeBestEffortOrNull({{ String(fromDate) }}) + {% end %} + {% if defined(toDate) %} + AND toDateTime(timestamp / 1000) <= parseDateTimeBestEffortOrNull({{ String(toDate) }}) + {% end %} + GROUP BY h + ORDER BY h DESC + + +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_latency_1d_multi__v1.pipe b/packages/tinybird/endpoints/endpoint__dns_metrics_regions_14d__v0.pipe similarity index 67% rename from packages/tinybird/pipes/endpoint__tcp_metrics_latency_1d_multi__v1.pipe rename to packages/tinybird/endpoints/endpoint__dns_metrics_regions_14d__v0.pipe index 791d98972..b34f37df0 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_latency_1d_multi__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_metrics_regions_14d__v0.pipe @@ -1,24 +1,26 @@ -TAGS "tcp" +TAGS "dns" NODE endpoint SQL > % SELECT + region, toStartOfInterval( toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, - monitorId, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, round(quantile(0.75)(latency)) as p75Latency, round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__tcp_1d__v0 + FROM dns_response__v0 WHERE - monitorId IN {{ Array(monitorIds, 'String', '4433') }} - GROUP BY h, monitorId + monitorId = {{ String(monitorId, '7417', required=True) }} + {% if defined(regions) %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} + GROUP BY h, region ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__dns_status_45d__v0.pipe b/packages/tinybird/endpoints/endpoint__dns_status_45d__v0.pipe similarity index 96% rename from packages/tinybird/pipes/endpoint__dns_status_45d__v0.pipe rename to packages/tinybird/endpoints/endpoint__dns_status_45d__v0.pipe index 55abcbc01..49376ca4f 100644 --- a/packages/tinybird/pipes/endpoint__dns_status_45d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_status_45d__v0.pipe @@ -17,3 +17,4 @@ SQL > ORDER BY day DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__dns_uptime_30d__v0.pipe b/packages/tinybird/endpoints/endpoint__dns_uptime_30d__v0.pipe similarity index 98% rename from packages/tinybird/pipes/endpoint__dns_uptime_30d__v0.pipe rename to packages/tinybird/endpoints/endpoint__dns_uptime_30d__v0.pipe index a0fca9b4d..183e244ab 100644 --- a/packages/tinybird/pipes/endpoint__dns_uptime_30d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_uptime_30d__v0.pipe @@ -25,3 +25,4 @@ SQL > ORDER BY interval DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_get_14d__v0.pipe b/packages/tinybird/endpoints/endpoint__http_get_14d__v0.pipe similarity index 94% rename from packages/tinybird/pipes/endpoint__http_get_14d__v0.pipe rename to packages/tinybird/endpoints/endpoint__http_get_14d__v0.pipe index 046358b6a..51a913609 100644 --- a/packages/tinybird/pipes/endpoint__http_get_14d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__http_get_14d__v0.pipe @@ -12,3 +12,4 @@ SQL > ORDER BY time DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_get_30d.pipe b/packages/tinybird/endpoints/endpoint__http_get_30d.pipe similarity index 84% rename from packages/tinybird/pipes/endpoint__tcp_get_30d.pipe rename to packages/tinybird/endpoints/endpoint__http_get_30d.pipe index b4b1e59dd..2323a7e74 100644 --- a/packages/tinybird/pipes/endpoint__tcp_get_30d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_get_30d.pipe @@ -1,13 +1,13 @@ VERSION 0 -TAGS tcp +TAGS http NODE endpoint SQL > % SELECT * - FROM mv__tcp_full_30d__v0 + FROM mv__http_full_30d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} AND cronTimestamp = {{ Int64(cronTimestamp, 1709477432205, required=True) }} @@ -16,3 +16,4 @@ SQL > +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_list_14d.pipe b/packages/tinybird/endpoints/endpoint__http_list_14d.pipe similarity index 71% rename from packages/tinybird/pipes/endpoint__tcp_list_14d.pipe rename to packages/tinybird/endpoints/endpoint__http_list_14d.pipe index 490123de0..52040df31 100644 --- a/packages/tinybird/pipes/endpoint__tcp_list_14d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_list_14d.pipe @@ -1,14 +1,15 @@ VERSION 0 -TAGS tcp +TAGS http NODE endpoint SQL > % - SELECT * FROM mv__tcp_14d__v0 + SELECT * FROM mv__http_14d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} ORDER BY cronTimestamp DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_list_1d__v1.pipe b/packages/tinybird/endpoints/endpoint__http_list_14d__v1.pipe similarity index 89% rename from packages/tinybird/pipes/endpoint__http_list_1d__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_list_14d__v1.pipe index 343a4f0cc..66d1b4ab5 100644 --- a/packages/tinybird/pipes/endpoint__http_list_1d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_list_14d__v1.pipe @@ -4,7 +4,7 @@ NODE endpoint SQL > % - SELECT * FROM mv__http_1d__v1 + SELECT * FROM mv__http_14d__v1 WHERE monitorId = {{ String(monitorId, '1', required=True) }} {% if defined(fromDate) %} @@ -16,3 +16,4 @@ SQL > ORDER BY time DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_list_14d.pipe b/packages/tinybird/endpoints/endpoint__http_list_1d.pipe similarity index 76% rename from packages/tinybird/pipes/endpoint__http_list_14d.pipe rename to packages/tinybird/endpoints/endpoint__http_list_1d.pipe index 21f6d74fa..195642841 100644 --- a/packages/tinybird/pipes/endpoint__http_list_14d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_list_1d.pipe @@ -6,9 +6,10 @@ NODE endpoint SQL > % - SELECT * FROM mv__http_14d__v0 + SELECT * FROM mv__http_1d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} ORDER BY cronTimestamp DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_list_1d__v1.pipe b/packages/tinybird/endpoints/endpoint__http_list_1d__v1.pipe similarity index 74% rename from packages/tinybird/pipes/endpoint__tcp_list_1d__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_list_1d__v1.pipe index 412f0503a..9537927d2 100644 --- a/packages/tinybird/pipes/endpoint__tcp_list_1d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_list_1d__v1.pipe @@ -1,12 +1,12 @@ -TAGS "tcp" +TAGS "http" NODE endpoint SQL > % - SELECT * FROM mv__tcp_1d__v1 + SELECT * FROM mv__http_1d__v1 WHERE - monitorId = {{ String(monitorId, '4433', required=True) }} + monitorId = {{ String(monitorId, '1', required=True) }} {% if defined(fromDate) %} AND time >= toDateTime(fromUnixTimestamp64Milli(toInt64({{ String(fromDate) }}))) {% end %} @@ -16,3 +16,4 @@ SQL > ORDER BY time DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_list_1d.pipe b/packages/tinybird/endpoints/endpoint__http_list_7d.pipe similarity index 71% rename from packages/tinybird/pipes/endpoint__tcp_list_1d.pipe rename to packages/tinybird/endpoints/endpoint__http_list_7d.pipe index 4f63aa1a9..827af3434 100644 --- a/packages/tinybird/pipes/endpoint__tcp_list_1d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_list_7d.pipe @@ -1,14 +1,15 @@ VERSION 0 -TAGS tcp +TAGS http NODE endpoint SQL > % - SELECT * FROM mv__tcp_1d__v0 + SELECT * FROM mv__http_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} ORDER BY cronTimestamp DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_list_7d__v1.pipe b/packages/tinybird/endpoints/endpoint__http_list_7d__v1.pipe similarity index 97% rename from packages/tinybird/pipes/endpoint__http_list_7d__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_list_7d__v1.pipe index 377104b7a..934891dce 100644 --- a/packages/tinybird/pipes/endpoint__http_list_7d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_list_7d__v1.pipe @@ -16,3 +16,4 @@ SQL > ORDER BY time DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_1d.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_14d.pipe similarity index 77% rename from packages/tinybird/pipes/endpoint__tcp_metrics_1d.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_14d.pipe index a1e600433..f32a91a73 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_1d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_14d.pipe @@ -1,6 +1,6 @@ VERSION 0 -TAGS tcp +TAGS http NODE endpoint SQL > @@ -15,10 +15,10 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, max(cronTimestamp) AS lastTimestamp - FROM mv__tcp_1d__v0 + FROM mv__http_14d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 1 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) UNION ALL SELECT round(quantile(0.50)(latency)) AS p50Latency, @@ -29,10 +29,11 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__tcp_7d__v0 -- REMINDER: this will increase the processed data compared to 3d + FROM mv__http_30d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 2 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 1 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 28 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 14 DAY, 3) +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__dns_metrics_7d__v0.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_14d__v1.pipe similarity index 74% rename from packages/tinybird/pipes/endpoint__dns_metrics_7d__v0.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_14d__v1.pipe index f89fe1702..f3e47727f 100644 --- a/packages/tinybird/pipes/endpoint__dns_metrics_7d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_14d__v1.pipe @@ -1,4 +1,4 @@ -TAGS "dns" +TAGS "http" NODE endpoint SQL > @@ -15,10 +15,10 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, max(cronTimestamp) AS lastTimestamp - FROM dns_response__v0 + FROM mv__http_14d__v1 WHERE - monitorId = {{ String(monitorId, '7417', required=True) }} - AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 7 DAY, 3)) + monitorId = {{ String(monitorId, '1', required=True) }} + AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} UNION ALL SELECT @@ -32,11 +32,11 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM dns_response__v0 + FROM mv__http_30d__v1 WHERE - monitorId = {{ String(monitorId, '7417', required=True) }} - AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 14 DAY, 3)) - AND timestamp < toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 7 DAY, 3)) + monitorId = {{ String(monitorId, '1', required=True) }} + AND time >= toDateTime64(now() - INTERVAL 28 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 14 DAY, 3) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} - +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_14d.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_1d.pipe similarity index 66% rename from packages/tinybird/pipes/endpoint__tcp_metrics_14d.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_1d.pipe index 9fdf8db5c..cf79d736b 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_14d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_1d.pipe @@ -1,6 +1,6 @@ VERSION 0 -TAGS tcp +TAGS http NODE endpoint SQL > @@ -15,13 +15,10 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, max(cronTimestamp) AS lastTimestamp - FROM mv__tcp_14d__v0 + FROM mv__http_1d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - -- FIXME: we can reduce the data processed by using removing it entirely - -- because the query is useless as we are in the 14d context - -- TODO: check where we can reduce the data processed - AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 1 DAY, 3) UNION ALL SELECT round(quantile(0.50)(latency)) AS p50Latency, @@ -32,10 +29,11 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__tcp_30d__v0 -- REMINDER: this will increase the processed data compared to 3d + FROM mv__http_7d__v0 -- REMINDER: this will increase the processed data compared to 3d WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 28 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 14 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 2 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 1 DAY, 3) +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__dns_metrics_1d__v0.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_1d__v1.pipe similarity index 74% rename from packages/tinybird/pipes/endpoint__dns_metrics_1d__v0.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_1d__v1.pipe index abaf8d083..db605bf16 100644 --- a/packages/tinybird/pipes/endpoint__dns_metrics_1d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_1d__v1.pipe @@ -1,4 +1,4 @@ -TAGS "dns" +TAGS "http" NODE endpoint SQL > @@ -15,10 +15,10 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, max(cronTimestamp) AS lastTimestamp - FROM dns_response__v0 + FROM mv__http_1d__v1 WHERE - monitorId = {{ String(monitorId, '7417', required=True) }} - AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 1 DAY, 3)) + monitorId = {{ String(monitorId, '1', required=True) }} + AND time >= toDateTime64(now() - INTERVAL 1 DAY, 3) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} UNION ALL SELECT @@ -32,11 +32,12 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM dns_response__v0 + FROM mv__http_7d__v1 WHERE - monitorId = {{ String(monitorId, '7417', required=True) }} - AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 2 DAY, 3)) - AND timestamp < toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 1 DAY, 3)) + monitorId = {{ String(monitorId, '1', required=True) }} + AND time >= toDateTime64(now() - INTERVAL 2 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 1 DAY, 3) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_14d.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_7d.pipe similarity index 81% rename from packages/tinybird/pipes/endpoint__http_metrics_14d.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_7d.pipe index b2526a18f..fe7448e38 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_14d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_7d.pipe @@ -15,10 +15,10 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, max(cronTimestamp) AS lastTimestamp - FROM mv__http_14d__v0 + FROM mv__http_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 7 DAY, 3) UNION ALL SELECT round(quantile(0.50)(latency)) AS p50Latency, @@ -29,10 +29,11 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__http_30d__v0 + FROM mv__http_14d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 28 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 14 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 7 DAY, 3) +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_7d__v1.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_7d__v1.pipe similarity index 99% rename from packages/tinybird/pipes/endpoint__http_metrics_7d__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_7d__v1.pipe index 25cd0ada0..06540408f 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_7d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_7d__v1.pipe @@ -39,4 +39,4 @@ SQL > AND time < toDateTime64(now() - INTERVAL 7 DAY, 3) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} - +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_latency_1d__v1.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_by_interval_14d.pipe similarity index 57% rename from packages/tinybird/pipes/endpoint__tcp_metrics_latency_1d__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_by_interval_14d.pipe index 6ef3e2260..a738bbcba 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_latency_1d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_by_interval_14d.pipe @@ -1,12 +1,16 @@ -TAGS "tcp" +VERSION 0 + +TAGS http NODE endpoint SQL > % SELECT + region, toStartOfInterval( - toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE + toDateTime(cronTimestamp / 1000), + INTERVAL {{ Int64(interval, 30) }} MINUTE -- use 2880 (2d) in case you want the 1d summary for the regions ) as h, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, @@ -14,10 +18,11 @@ SQL > round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__tcp_1d__v0 + FROM mv__http_14d__v0 WHERE - monitorId = {{ String(monitorId, '4433', required=True) }} - GROUP BY h + monitorId = {{ String(monitorId, '1', required=True) }} + GROUP BY h, region ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_regions_7d__v0.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_by_interval_1d.pipe similarity index 70% rename from packages/tinybird/pipes/endpoint__http_metrics_regions_7d__v0.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_by_interval_1d.pipe index 21a89f6e8..3ea88c6a3 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_regions_7d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_by_interval_1d.pipe @@ -1,3 +1,7 @@ +VERSION 0 + +TAGS http + NODE endpoint SQL > @@ -5,7 +9,8 @@ SQL > SELECT region, toStartOfInterval( - toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE + toDateTime(cronTimestamp / 1000), + INTERVAL {{ Int64(interval, 30) }} MINUTE -- use 2880 (2d) in case you want the 1d summary for the regions ) as h, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, @@ -13,11 +18,11 @@ SQL > round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__http_7d__v0 + FROM mv__http_1d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} GROUP BY h, region ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_by_interval_14d.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_by_interval_7d.pipe similarity index 94% rename from packages/tinybird/pipes/endpoint__http_metrics_by_interval_14d.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_by_interval_7d.pipe index 19288eefa..342494b1a 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_by_interval_14d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_by_interval_7d.pipe @@ -18,10 +18,11 @@ SQL > round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__http_14d__v0 + FROM mv__http_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} GROUP BY h, region ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_by_region_7d.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_by_region_14d.pipe similarity index 92% rename from packages/tinybird/pipes/endpoint__http_metrics_by_region_7d.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_by_region_14d.pipe index 2ea818314..2e985e4bd 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_by_region_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_by_region_14d.pipe @@ -15,10 +15,14 @@ SQL > round(quantile(0.99)(latency)) as p99Latency, count() as count, count(if(error = 0, 1, NULL)) AS ok - FROM mv__http_7d__v0 + FROM mv__http_14d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} GROUP BY region + + + +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_by_region_1d.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_by_region_1d.pipe similarity index 97% rename from packages/tinybird/pipes/endpoint__http_metrics_by_region_1d.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_by_region_1d.pipe index 1390411b0..28572163a 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_by_region_1d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_by_region_1d.pipe @@ -22,3 +22,4 @@ SQL > +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_by_region_14d.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_by_region_7d.pipe similarity index 92% rename from packages/tinybird/pipes/endpoint__http_metrics_by_region_14d.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_by_region_7d.pipe index e4451473b..dfe49ac29 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_by_region_14d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_by_region_7d.pipe @@ -15,13 +15,11 @@ SQL > round(quantile(0.99)(latency)) as p99Latency, count() as count, count(if(error = 0, 1, NULL)) AS ok - FROM mv__http_14d__v0 + FROM mv__http_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} GROUP BY region - - - +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_global_1d.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_global_1d__v0.pipe similarity index 82% rename from packages/tinybird/pipes/endpoint__tcp_metrics_global_1d.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_global_1d__v0.pipe index 8742e3a6a..b8448c7d2 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_global_1d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_global_1d__v0.pipe @@ -1,6 +1,6 @@ VERSION 0 -TAGS "tcp" +TAGS "http" NODE endpoint SQL > @@ -17,8 +17,9 @@ SQL > max(cronTimestamp) as lastTimestamp, count() as count, monitorId - FROM mv__tcp_1d__v0 - WHERE monitorId IN {{ Array(monitorIds, 'String', '4433') }} + FROM mv__http_1d__v0 + WHERE monitorId IN {{ Array(monitorIds, 'String', '1') }} GROUP BY monitorId +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_by_interval_7d.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_latency_1d__v1.pipe similarity index 67% rename from packages/tinybird/pipes/endpoint__http_metrics_by_interval_7d.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_latency_1d__v1.pipe index e8daaf42a..f6d0e812e 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_by_interval_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_latency_1d__v1.pipe @@ -1,16 +1,12 @@ -VERSION 0 - -TAGS http +TAGS "tcp" NODE endpoint SQL > % SELECT - region, toStartOfInterval( - toDateTime(cronTimestamp / 1000), - INTERVAL {{ Int64(interval, 30) }} MINUTE -- use 2880 (2d) in case you want the 1d summary for the regions + toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, @@ -18,10 +14,11 @@ SQL > round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__http_7d__v0 + FROM mv__http_1d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - GROUP BY h, region + GROUP BY h ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__dns_metrics_latency_7d__v0.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_latency_1d_multi__v1.pipe similarity index 53% rename from packages/tinybird/pipes/endpoint__dns_metrics_latency_7d__v0.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_latency_1d_multi__v1.pipe index 5cc7a412d..94dd17262 100644 --- a/packages/tinybird/pipes/endpoint__dns_metrics_latency_7d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_latency_1d_multi__v1.pipe @@ -1,4 +1,4 @@ -TAGS "dns" +TAGS "http" NODE endpoint SQL > @@ -8,22 +8,18 @@ SQL > toStartOfInterval( toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, + monitorId, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, round(quantile(0.75)(latency)) as p75Latency, round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM dns_response__v0 + FROM mv__http_1d__v0 WHERE - monitorId = {{ String(monitorId, '7417', required=True) }} - {% if defined(fromDate) %} - AND toDateTime(timestamp / 1000) >= parseDateTimeBestEffortOrNull({{ String(fromDate) }}) - {% end %} - {% if defined(toDate) %} - AND toDateTime(timestamp / 1000) <= parseDateTimeBestEffortOrNull({{ String(toDate) }}) - {% end %} - GROUP BY h - ORDER BY h DESC + monitorId IN {{ Array(monitorIds, 'String', '1,666') }} + GROUP BY h, monitorId + ORDER BY h ASC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__dns_metrics_regions_14d__v0.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_latency_7d__v1.pipe similarity index 68% rename from packages/tinybird/pipes/endpoint__dns_metrics_regions_14d__v0.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_latency_7d__v1.pipe index dd60f8ffe..be6945998 100644 --- a/packages/tinybird/pipes/endpoint__dns_metrics_regions_14d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_latency_7d__v1.pipe @@ -1,11 +1,10 @@ -TAGS "dns" +TAGS "tcp" NODE endpoint SQL > % SELECT - region, toStartOfInterval( toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, @@ -15,11 +14,11 @@ SQL > round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM dns_response__v0 + FROM mv__http_7d__v0 WHERE - monitorId = {{ String(monitorId, '7417', required=True) }} - {% if defined(regions) %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} - GROUP BY h, region + monitorId = {{ String(monitorId, '1', required=True) }} + GROUP BY h ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_regions_1d__v0.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_regions_14d__v0.pipe similarity index 94% rename from packages/tinybird/pipes/endpoint__http_metrics_regions_1d__v0.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_regions_14d__v0.pipe index 162a6bf69..1a50f28c0 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_regions_1d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_regions_14d__v0.pipe @@ -13,7 +13,7 @@ SQL > round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__http_1d__v0 + FROM mv__http_14d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} @@ -21,3 +21,4 @@ SQL > ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_latency_1d_multi__v1.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_regions_1d__v0.pipe similarity index 70% rename from packages/tinybird/pipes/endpoint__http_metrics_latency_1d_multi__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_regions_1d__v0.pipe index 7adac3fb6..b7eb47955 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_latency_1d_multi__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_regions_1d__v0.pipe @@ -1,14 +1,12 @@ -TAGS "http" - NODE endpoint SQL > % SELECT + region, toStartOfInterval( toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, - monitorId, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, round(quantile(0.75)(latency)) as p75Latency, @@ -17,8 +15,10 @@ SQL > round(quantile(0.99)(latency)) as p99Latency FROM mv__http_1d__v0 WHERE - monitorId IN {{ Array(monitorIds, 'String', '1,666') }} - GROUP BY h, monitorId - ORDER BY h ASC + monitorId = {{ String(monitorId, '1', required=True) }} + {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} + GROUP BY h, region + ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_latency_7d__v1.pipe b/packages/tinybird/endpoints/endpoint__http_metrics_regions_7d__v0.pipe similarity index 81% rename from packages/tinybird/pipes/endpoint__http_metrics_latency_7d__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_metrics_regions_7d__v0.pipe index 2cd927ee0..6a42de62b 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_latency_7d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_metrics_regions_7d__v0.pipe @@ -1,10 +1,9 @@ -TAGS "tcp" - NODE endpoint SQL > % SELECT + region, toStartOfInterval( toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, @@ -17,7 +16,9 @@ SQL > FROM mv__http_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - GROUP BY h + {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} + GROUP BY h, region ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_status_14d.pipe b/packages/tinybird/endpoints/endpoint__http_status_14d.pipe similarity index 97% rename from packages/tinybird/pipes/endpoint__http_status_14d.pipe rename to packages/tinybird/endpoints/endpoint__http_status_14d.pipe index a384e6564..4e18069aa 100644 --- a/packages/tinybird/pipes/endpoint__http_status_14d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_status_14d.pipe @@ -23,3 +23,4 @@ SQL > LIMIT {{ Int16(days, 14) }} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_status_7d.pipe b/packages/tinybird/endpoints/endpoint__http_status_45d.pipe similarity index 74% rename from packages/tinybird/pipes/endpoint__tcp_status_7d.pipe rename to packages/tinybird/endpoints/endpoint__http_status_45d.pipe index f5fc07d99..b297fdc5c 100644 --- a/packages/tinybird/pipes/endpoint__tcp_status_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_status_45d.pipe @@ -1,13 +1,13 @@ VERSION 0 -TAGS tcp +TAGS http NODE endpoint SQL > % SELECT time as day, countMerge(count) as count, countMerge(ok) as ok - FROM mv__tcp_status_7d__v0 + FROM mv__http_status_45d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} GROUP BY day @@ -16,7 +16,9 @@ SQL > FROM toStartOfDay(toStartOfDay(toTimeZone(now(), 'UTC'))) TO toStartOfDay( - date_sub(DAY, 7, now()) + date_sub(DAY, 45, now()) ) STEP INTERVAL -1 DAY + LIMIT {{ Int16(days, 45) }} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_status_45d__v1.pipe b/packages/tinybird/endpoints/endpoint__http_status_45d__v1.pipe similarity index 69% rename from packages/tinybird/pipes/endpoint__tcp_status_45d__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_status_45d__v1.pipe index 846d52a45..c6cdcb1e9 100644 --- a/packages/tinybird/pipes/endpoint__tcp_status_45d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_status_45d__v1.pipe @@ -1,4 +1,4 @@ -TAGS "tcp" +TAGS "http" NODE endpoint SQL > @@ -11,9 +11,10 @@ SQL > countMerge(success) as ok, countMerge(error) as error, countMerge(degraded) as degraded - FROM mv__tcp_status_45d__v1 - WHERE monitorId IN {{ Array(monitorIds, 'String', '4433') }} + FROM mv__http_status_45d__v1 + WHERE monitorId IN {{ Array(monitorIds, 'String', '1,666') }} GROUP BY day, monitorId ORDER BY day DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_status_45d.pipe b/packages/tinybird/endpoints/endpoint__http_status_7d.pipe similarity index 76% rename from packages/tinybird/pipes/endpoint__tcp_status_45d.pipe rename to packages/tinybird/endpoints/endpoint__http_status_7d.pipe index c0e3081d7..1f1c2a3fd 100644 --- a/packages/tinybird/pipes/endpoint__tcp_status_45d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_status_7d.pipe @@ -1,13 +1,13 @@ VERSION 0 -TAGS tcp +TAGS http NODE endpoint SQL > % SELECT time as day, countMerge(count) as count, countMerge(ok) as ok - FROM mv__tcp_status_45d__v0 + FROM mv__http_status_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} GROUP BY day @@ -16,8 +16,8 @@ SQL > FROM toStartOfDay(toStartOfDay(toTimeZone(now(), 'UTC'))) TO toStartOfDay( - date_sub(DAY, 45, now()) + date_sub(DAY, 7, now()) ) STEP INTERVAL -1 DAY - LIMIT {{ Int16(days, 45) }} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_timing_phases_14d__v1.pipe b/packages/tinybird/endpoints/endpoint__http_timing_phases_14d__v1.pipe similarity index 99% rename from packages/tinybird/pipes/endpoint__http_timing_phases_14d__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_timing_phases_14d__v1.pipe index 7d91fdb41..186b65e44 100644 --- a/packages/tinybird/pipes/endpoint__http_timing_phases_14d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_timing_phases_14d__v1.pipe @@ -36,3 +36,4 @@ SQL > ORDER BY h ASC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_uptime_30d.pipe b/packages/tinybird/endpoints/endpoint__http_uptime_30d__v1.pipe similarity index 67% rename from packages/tinybird/pipes/endpoint__http_uptime_30d.pipe rename to packages/tinybird/endpoints/endpoint__http_uptime_30d__v1.pipe index 79266ded6..44cf82611 100644 --- a/packages/tinybird/pipes/endpoint__http_uptime_30d.pipe +++ b/packages/tinybird/endpoints/endpoint__http_uptime_30d__v1.pipe @@ -1,12 +1,10 @@ -VERSION 1 - TAGS "http" NODE endpoint SQL > % - SELECT + SELECT toStartOfInterval(time, INTERVAL {{ String(interval, '30', required=True) }} minute) AS interval, countIf(requestStatus = 'success') AS success, countIf(requestStatus = 'degraded') AS degraded, @@ -14,8 +12,11 @@ SQL > FROM mv__http_uptime_30d__v1 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - {% if fromDate %} AND time >= toDateTime({{ String(fromDate) }}) {% end %} - {% if toDate %} AND time <= toDateTime({{ String(toDate) }}) {% end %} + {% if fromDate %} AND time >= parseDateTimeBestEffortOrNull({{ String(fromDate) }}) {% end %} + {% if toDate %} AND time <= parseDateTimeBestEffortOrNull({{ String(toDate) }}) {% end %} {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} GROUP BY interval - ORDER BY interval DESC \ No newline at end of file + ORDER BY interval DESC + + +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_uptime_7d__v1.pipe b/packages/tinybird/endpoints/endpoint__http_uptime_7d__v1.pipe similarity index 84% rename from packages/tinybird/pipes/endpoint__tcp_uptime_7d__v1.pipe rename to packages/tinybird/endpoints/endpoint__http_uptime_7d__v1.pipe index 5815b90f5..357de6661 100644 --- a/packages/tinybird/pipes/endpoint__tcp_uptime_7d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__http_uptime_7d__v1.pipe @@ -4,14 +4,14 @@ NODE endpoint SQL > % - SELECT + SELECT toStartOfInterval(time, INTERVAL {{ String(interval, '30', required=True) }} minute) AS interval, countIf(requestStatus = 'success') AS success, countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error - FROM mv__tcp_uptime_7d__v1 + FROM mv__http_uptime_7d__v1 WHERE - monitorId = {{ String(monitorId, '4433', required=True) }} + monitorId = {{ String(monitorId, '1', required=True) }} {% if fromDate %} AND time >= parseDateTimeBestEffortOrNull({{ String(fromDate) }}) {% end %} {% if toDate %} AND time <= parseDateTimeBestEffortOrNull({{ String(toDate) }}) {% end %} {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} @@ -19,3 +19,4 @@ SQL > ORDER BY interval DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_workspace_30d__v0.pipe b/packages/tinybird/endpoints/endpoint__http_workspace_30d__v0.pipe similarity index 94% rename from packages/tinybird/pipes/endpoint__http_workspace_30d__v0.pipe rename to packages/tinybird/endpoints/endpoint__http_workspace_30d__v0.pipe index 67149c4ff..028cbd22d 100644 --- a/packages/tinybird/pipes/endpoint__http_workspace_30d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__http_workspace_30d__v0.pipe @@ -13,3 +13,4 @@ SQL > ORDER BY day DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__stats_global.pipe b/packages/tinybird/endpoints/endpoint__stats_global.pipe similarity index 98% rename from packages/tinybird/pipes/endpoint__stats_global.pipe rename to packages/tinybird/endpoints/endpoint__stats_global.pipe index ed4b7c235..4e68659ea 100644 --- a/packages/tinybird/pipes/endpoint__stats_global.pipe +++ b/packages/tinybird/endpoints/endpoint__stats_global.pipe @@ -22,3 +22,4 @@ SQL > {% end %} {% end %} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_get_14d__v0.pipe b/packages/tinybird/endpoints/endpoint__tcp_get_14d__v0.pipe similarity index 94% rename from packages/tinybird/pipes/endpoint__tcp_get_14d__v0.pipe rename to packages/tinybird/endpoints/endpoint__tcp_get_14d__v0.pipe index 961bd48ad..9f012b53f 100644 --- a/packages/tinybird/pipes/endpoint__tcp_get_14d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_get_14d__v0.pipe @@ -12,3 +12,4 @@ SQL > ORDER BY time DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_get_30d.pipe b/packages/tinybird/endpoints/endpoint__tcp_get_30d.pipe similarity index 85% rename from packages/tinybird/pipes/endpoint__http_get_30d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_get_30d.pipe index b9405fb0e..a3812df14 100644 --- a/packages/tinybird/pipes/endpoint__http_get_30d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_get_30d.pipe @@ -1,13 +1,13 @@ VERSION 0 -TAGS http +TAGS tcp NODE endpoint SQL > % SELECT * - FROM mv__http_full_30d__v0 + FROM mv__tcp_full_30d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} AND cronTimestamp = {{ Int64(cronTimestamp, 1709477432205, required=True) }} @@ -16,3 +16,4 @@ SQL > +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_list_7d.pipe b/packages/tinybird/endpoints/endpoint__tcp_list_14d.pipe similarity index 76% rename from packages/tinybird/pipes/endpoint__tcp_list_7d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_list_14d.pipe index 88e043795..c3f42e52f 100644 --- a/packages/tinybird/pipes/endpoint__tcp_list_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_list_14d.pipe @@ -6,9 +6,10 @@ NODE endpoint SQL > % - SELECT * FROM mv__tcp_7d__v0 + SELECT * FROM mv__tcp_14d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} ORDER BY cronTimestamp DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_list_7d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_list_14d__v1.pipe similarity index 89% rename from packages/tinybird/pipes/endpoint__tcp_list_7d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_list_14d__v1.pipe index 33adaffe3..36b1f34ec 100644 --- a/packages/tinybird/pipes/endpoint__tcp_list_7d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_list_14d__v1.pipe @@ -4,7 +4,7 @@ NODE endpoint SQL > % - SELECT * FROM mv__tcp_7d__v1 + SELECT * FROM mv__tcp_14d__v1 WHERE monitorId = {{ String(monitorId, '4433', required=True) }} {% if defined(fromDate) %} @@ -16,3 +16,4 @@ SQL > ORDER BY time DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_list_1d.pipe b/packages/tinybird/endpoints/endpoint__tcp_list_1d.pipe similarity index 72% rename from packages/tinybird/pipes/endpoint__http_list_1d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_list_1d.pipe index 396ae8bd2..2d38ba1dc 100644 --- a/packages/tinybird/pipes/endpoint__http_list_1d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_list_1d.pipe @@ -1,14 +1,15 @@ VERSION 0 -TAGS http +TAGS tcp NODE endpoint SQL > % - SELECT * FROM mv__http_1d__v0 + SELECT * FROM mv__tcp_1d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} ORDER BY cronTimestamp DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_list_14d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_list_1d__v1.pipe similarity index 74% rename from packages/tinybird/pipes/endpoint__http_list_14d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_list_1d__v1.pipe index a05f372b0..fd1f5c2e7 100644 --- a/packages/tinybird/pipes/endpoint__http_list_14d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_list_1d__v1.pipe @@ -1,12 +1,12 @@ -TAGS "http" +TAGS "tcp" NODE endpoint SQL > % - SELECT * FROM mv__http_14d__v1 + SELECT * FROM mv__tcp_1d__v1 WHERE - monitorId = {{ String(monitorId, '1', required=True) }} + monitorId = {{ String(monitorId, '4433', required=True) }} {% if defined(fromDate) %} AND time >= toDateTime(fromUnixTimestamp64Milli(toInt64({{ String(fromDate) }}))) {% end %} @@ -16,3 +16,4 @@ SQL > ORDER BY time DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_list_7d.pipe b/packages/tinybird/endpoints/endpoint__tcp_list_7d.pipe similarity index 72% rename from packages/tinybird/pipes/endpoint__http_list_7d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_list_7d.pipe index 6df42e341..a11c24330 100644 --- a/packages/tinybird/pipes/endpoint__http_list_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_list_7d.pipe @@ -1,14 +1,15 @@ VERSION 0 -TAGS http +TAGS tcp NODE endpoint SQL > % - SELECT * FROM mv__http_7d__v0 + SELECT * FROM mv__tcp_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} ORDER BY cronTimestamp DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_list_14d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_list_7d__v1.pipe similarity index 90% rename from packages/tinybird/pipes/endpoint__tcp_list_14d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_list_7d__v1.pipe index 5c6341bd7..def193866 100644 --- a/packages/tinybird/pipes/endpoint__tcp_list_14d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_list_7d__v1.pipe @@ -4,7 +4,7 @@ NODE endpoint SQL > % - SELECT * FROM mv__tcp_14d__v1 + SELECT * FROM mv__tcp_7d__v1 WHERE monitorId = {{ String(monitorId, '4433', required=True) }} {% if defined(fromDate) %} @@ -16,3 +16,4 @@ SQL > ORDER BY time DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_7d.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_14d.pipe similarity index 66% rename from packages/tinybird/pipes/endpoint__http_metrics_7d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_14d.pipe index 482638b11..6eb070952 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_14d.pipe @@ -1,6 +1,6 @@ VERSION 0 -TAGS http +TAGS tcp NODE endpoint SQL > @@ -15,10 +15,13 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, max(cronTimestamp) AS lastTimestamp - FROM mv__http_7d__v0 + FROM mv__tcp_14d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 7 DAY, 3) + -- FIXME: we can reduce the data processed by using removing it entirely + -- because the query is useless as we are in the 14d context + -- TODO: check where we can reduce the data processed + AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) UNION ALL SELECT round(quantile(0.50)(latency)) AS p50Latency, @@ -29,10 +32,11 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__http_14d__v0 + FROM mv__tcp_30d__v0 -- REMINDER: this will increase the processed data compared to 3d WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 7 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 28 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 14 DAY, 3) +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__dns_metrics_14d__v0.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_14d__v1.pipe similarity index 74% rename from packages/tinybird/pipes/endpoint__dns_metrics_14d__v0.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_14d__v1.pipe index a22cac1cb..948a2e60d 100644 --- a/packages/tinybird/pipes/endpoint__dns_metrics_14d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_14d__v1.pipe @@ -1,4 +1,4 @@ -TAGS "dns" +TAGS "tcp" NODE endpoint SQL > @@ -15,10 +15,10 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, max(cronTimestamp) AS lastTimestamp - FROM dns_response__v0 + FROM mv__tcp_14d__v1 WHERE - monitorId = {{ String(monitorId, '7417', required=True) }} - AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 14 DAY, 3)) + monitorId = {{ String(monitorId, '4433', required=True) }} + AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} UNION ALL SELECT @@ -32,11 +32,12 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM dns_response__v0 + FROM mv__tcp_30d__v1 WHERE - monitorId = {{ String(monitorId, '7417', required=True) }} - AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 28 DAY, 3)) - AND timestamp < toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 14 DAY, 3)) + monitorId = {{ String(monitorId, '4433', required=True) }} + AND time >= toDateTime64(now() - INTERVAL 28 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 14 DAY, 3) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_7d.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_1d.pipe similarity index 77% rename from packages/tinybird/pipes/endpoint__tcp_metrics_7d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_1d.pipe index 3a009c1de..6eaeac95d 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_1d.pipe @@ -15,10 +15,10 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, max(cronTimestamp) AS lastTimestamp - FROM mv__tcp_7d__v0 + FROM mv__tcp_1d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 7 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 1 DAY, 3) UNION ALL SELECT round(quantile(0.50)(latency)) AS p50Latency, @@ -29,10 +29,11 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__tcp_14d__v0 -- REMINDER: this will increase the processed data compared to 3d + FROM mv__tcp_7d__v0 -- REMINDER: this will increase the processed data compared to 3d WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 7 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 2 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 1 DAY, 3) +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_1d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_1d__v1.pipe similarity index 99% rename from packages/tinybird/pipes/endpoint__tcp_metrics_1d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_1d__v1.pipe index a2be394f2..adc2c9a94 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_1d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_1d__v1.pipe @@ -40,3 +40,4 @@ SQL > {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_1d.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_7d.pipe similarity index 76% rename from packages/tinybird/pipes/endpoint__http_metrics_1d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_7d.pipe index 299302e99..061bb0222 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_1d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_7d.pipe @@ -1,6 +1,6 @@ VERSION 0 -TAGS http +TAGS tcp NODE endpoint SQL > @@ -15,10 +15,10 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, max(cronTimestamp) AS lastTimestamp - FROM mv__http_1d__v0 + FROM mv__tcp_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 1 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 7 DAY, 3) UNION ALL SELECT round(quantile(0.50)(latency)) AS p50Latency, @@ -29,10 +29,11 @@ SQL > count() as count, count(if(error = 0, 1, NULL)) AS ok, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__http_7d__v0 -- REMINDER: this will increase the processed data compared to 3d + FROM mv__tcp_14d__v0 -- REMINDER: this will increase the processed data compared to 3d WHERE monitorId = {{ String(monitorId, '1', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 2 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 1 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 7 DAY, 3) +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_14d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_7d__v1.pipe similarity index 85% rename from packages/tinybird/pipes/endpoint__tcp_metrics_14d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_7d__v1.pipe index d9fbd9c81..e5193ad40 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_14d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_7d__v1.pipe @@ -15,10 +15,10 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, max(cronTimestamp) AS lastTimestamp - FROM mv__tcp_14d__v1 + FROM mv__tcp_7d__v1 WHERE monitorId = {{ String(monitorId, '4433', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 7 DAY, 3) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} UNION ALL SELECT @@ -32,11 +32,12 @@ SQL > countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error, NULL as lastTimestamp -- no need to query the `lastTimestamp` as not relevant - FROM mv__tcp_30d__v1 + FROM mv__tcp_14d__v1 WHERE monitorId = {{ String(monitorId, '4433', required=True) }} - AND time >= toDateTime64(now() - INTERVAL 28 DAY, 3) - AND time < toDateTime64(now() - INTERVAL 14 DAY, 3) + AND time >= toDateTime64(now() - INTERVAL 14 DAY, 3) + AND time < toDateTime64(now() - INTERVAL 7 DAY, 3) {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__dns_metrics_latency_1d_multi__v0.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_interval_14d.pipe similarity index 55% rename from packages/tinybird/pipes/endpoint__dns_metrics_latency_1d_multi__v0.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_by_interval_14d.pipe index 4be7720d9..6bb2d8bc3 100644 --- a/packages/tinybird/pipes/endpoint__dns_metrics_latency_1d_multi__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_interval_14d.pipe @@ -1,25 +1,28 @@ -TAGS "dns" +VERSION 0 + +TAGS tcp NODE endpoint SQL > % SELECT + region, toStartOfInterval( - toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE + toDateTime(cronTimestamp / 1000), + INTERVAL {{ Int64(interval, 30) }} MINUTE -- use 2880 (2d) in case you want the 1d summary for the regions ) as h, - monitorId, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, round(quantile(0.75)(latency)) as p75Latency, round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM dns_response__v0 + FROM mv__tcp_14d__v0 WHERE - monitorId IN {{ Array(monitorIds, 'String', '7417') }} - AND timestamp >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 1 DAY, 3)) - GROUP BY h, monitorId - ORDER BY h ASC + monitorId = {{ String(monitorId, '1', required=True) }} + GROUP BY h, region + ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_by_interval_1d.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_interval_1d.pipe similarity index 98% rename from packages/tinybird/pipes/endpoint__tcp_metrics_by_interval_1d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_by_interval_1d.pipe index 6ce080204..9e1ecdc2b 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_by_interval_1d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_interval_1d.pipe @@ -25,3 +25,4 @@ SQL > ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_latency_1d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_interval_7d.pipe similarity index 66% rename from packages/tinybird/pipes/endpoint__http_metrics_latency_1d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_by_interval_7d.pipe index 59ee7cf68..8affe8df4 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_latency_1d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_interval_7d.pipe @@ -1,12 +1,16 @@ -TAGS "tcp" +VERSION 0 + +TAGS tcp NODE endpoint SQL > % SELECT + region, toStartOfInterval( - toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE + toDateTime(cronTimestamp / 1000), + INTERVAL {{ Int64(interval, 30) }} MINUTE -- use 2880 (2d) in case you want the 1d summary for the regions ) as h, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, @@ -14,10 +18,11 @@ SQL > round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__http_1d__v0 + FROM mv__tcp_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} - GROUP BY h + GROUP BY h, region ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_by_region_14d.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_region_14d.pipe similarity index 97% rename from packages/tinybird/pipes/endpoint__tcp_metrics_by_region_14d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_by_region_14d.pipe index dda3021d8..493ecd846 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_by_region_14d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_region_14d.pipe @@ -22,3 +22,4 @@ SQL > +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_by_region_1d.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_region_1d.pipe similarity index 97% rename from packages/tinybird/pipes/endpoint__tcp_metrics_by_region_1d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_by_region_1d.pipe index 44b93ca14..392f6f604 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_by_region_1d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_region_1d.pipe @@ -22,3 +22,4 @@ SQL > +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_by_region_7d.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_region_7d.pipe similarity index 97% rename from packages/tinybird/pipes/endpoint__tcp_metrics_by_region_7d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_by_region_7d.pipe index b0f59a2de..3b1cbaf03 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_by_region_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_by_region_7d.pipe @@ -22,3 +22,4 @@ SQL > +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_global_1d.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_global_1d.pipe similarity index 82% rename from packages/tinybird/pipes/endpoint__http_metrics_global_1d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_global_1d.pipe index 11fd2feed..2f65ff43c 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_global_1d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_global_1d.pipe @@ -1,6 +1,6 @@ VERSION 0 -TAGS "http" +TAGS "tcp" NODE endpoint SQL > @@ -17,8 +17,9 @@ SQL > max(cronTimestamp) as lastTimestamp, count() as count, monitorId - FROM mv__http_1d__v0 - WHERE monitorId IN {{ Array(monitorIds, 'String', '1') }} + FROM mv__tcp_1d__v0 + WHERE monitorId IN {{ Array(monitorIds, 'String', '4433') }} GROUP BY monitorId +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_by_interval_7d.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_latency_1d__v1.pipe similarity index 58% rename from packages/tinybird/pipes/endpoint__tcp_metrics_by_interval_7d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_latency_1d__v1.pipe index 143621973..12ac85423 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_by_interval_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_latency_1d__v1.pipe @@ -1,16 +1,12 @@ -VERSION 0 - -TAGS tcp +TAGS "tcp" NODE endpoint SQL > % SELECT - region, toStartOfInterval( - toDateTime(cronTimestamp / 1000), - INTERVAL {{ Int64(interval, 30) }} MINUTE -- use 2880 (2d) in case you want the 1d summary for the regions + toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, @@ -18,10 +14,11 @@ SQL > round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__tcp_7d__v0 + FROM mv__tcp_1d__v0 WHERE - monitorId = {{ String(monitorId, '1', required=True) }} - GROUP BY h, region + monitorId = {{ String(monitorId, '4433', required=True) }} + GROUP BY h ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_metrics_latency_7d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_latency_1d_multi__v1.pipe similarity index 78% rename from packages/tinybird/pipes/endpoint__tcp_metrics_latency_7d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_latency_1d_multi__v1.pipe index 108955e7b..032ea6ecd 100644 --- a/packages/tinybird/pipes/endpoint__tcp_metrics_latency_7d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_latency_1d_multi__v1.pipe @@ -8,16 +8,18 @@ SQL > toStartOfInterval( toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, + monitorId, toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, round(quantile(0.50)(latency)) as p50Latency, round(quantile(0.75)(latency)) as p75Latency, round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__tcp_7d__v1 + FROM mv__tcp_1d__v0 WHERE - monitorId = {{ String(monitorId, '4433', required=True) }} - GROUP BY h + monitorId IN {{ Array(monitorIds, 'String', '4433') }} + GROUP BY h, monitorId ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_metrics_regions_14d__v0.pipe b/packages/tinybird/endpoints/endpoint__tcp_metrics_latency_7d__v1.pipe similarity index 70% rename from packages/tinybird/pipes/endpoint__http_metrics_regions_14d__v0.pipe rename to packages/tinybird/endpoints/endpoint__tcp_metrics_latency_7d__v1.pipe index 7e48c4687..b8b418124 100644 --- a/packages/tinybird/pipes/endpoint__http_metrics_regions_14d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_metrics_latency_7d__v1.pipe @@ -1,9 +1,10 @@ +TAGS "tcp" + NODE endpoint SQL > % SELECT - region, toStartOfInterval( toDateTime(cronTimestamp / 1000), INTERVAL {{ Int64(interval, 30) }} MINUTE ) as h, @@ -13,11 +14,11 @@ SQL > round(quantile(0.90)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM mv__http_14d__v0 + FROM mv__tcp_7d__v1 WHERE - monitorId = {{ String(monitorId, '1', required=True) }} - {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} - GROUP BY h, region + monitorId = {{ String(monitorId, '4433', required=True) }} + GROUP BY h ORDER BY h DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_status_45d.pipe b/packages/tinybird/endpoints/endpoint__tcp_status_45d.pipe similarity index 88% rename from packages/tinybird/pipes/endpoint__http_status_45d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_status_45d.pipe index e77cab7ea..3f2f0b22c 100644 --- a/packages/tinybird/pipes/endpoint__http_status_45d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_status_45d.pipe @@ -1,13 +1,13 @@ VERSION 0 -TAGS http +TAGS tcp NODE endpoint SQL > % SELECT time as day, countMerge(count) as count, countMerge(ok) as ok - FROM mv__http_status_45d__v0 + FROM mv__tcp_status_45d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} GROUP BY day @@ -21,3 +21,4 @@ SQL > LIMIT {{ Int16(days, 45) }} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_status_45d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_status_45d__v1.pipe similarity index 69% rename from packages/tinybird/pipes/endpoint__http_status_45d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_status_45d__v1.pipe index 6a73ac5f6..5d1b35d1e 100644 --- a/packages/tinybird/pipes/endpoint__http_status_45d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_status_45d__v1.pipe @@ -1,4 +1,4 @@ -TAGS "http" +TAGS "tcp" NODE endpoint SQL > @@ -11,9 +11,10 @@ SQL > countMerge(success) as ok, countMerge(error) as error, countMerge(degraded) as degraded - FROM mv__http_status_45d__v1 - WHERE monitorId IN {{ Array(monitorIds, 'String', '1,666') }} + FROM mv__tcp_status_45d__v1 + WHERE monitorId IN {{ Array(monitorIds, 'String', '4433') }} GROUP BY day, monitorId ORDER BY day DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_status_7d.pipe b/packages/tinybird/endpoints/endpoint__tcp_status_7d.pipe similarity index 88% rename from packages/tinybird/pipes/endpoint__http_status_7d.pipe rename to packages/tinybird/endpoints/endpoint__tcp_status_7d.pipe index 9a01ad6a3..10169d50e 100644 --- a/packages/tinybird/pipes/endpoint__http_status_7d.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_status_7d.pipe @@ -1,13 +1,13 @@ VERSION 0 -TAGS http +TAGS tcp NODE endpoint SQL > % SELECT time as day, countMerge(count) as count, countMerge(ok) as ok - FROM mv__http_status_7d__v0 + FROM mv__tcp_status_7d__v0 WHERE monitorId = {{ String(monitorId, '1', required=True) }} GROUP BY day @@ -20,3 +20,4 @@ SQL > ) STEP INTERVAL -1 DAY +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_uptime_30d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_uptime_30d__v1.pipe similarity index 96% rename from packages/tinybird/pipes/endpoint__tcp_uptime_30d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_uptime_30d__v1.pipe index cac39d9ea..8218a8bad 100644 --- a/packages/tinybird/pipes/endpoint__tcp_uptime_30d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_uptime_30d__v1.pipe @@ -4,7 +4,7 @@ NODE endpoint SQL > % - SELECT + SELECT toStartOfInterval(time, INTERVAL {{ String(interval, '30', required=True) }} minute) AS interval, countIf(requestStatus = 'success') AS success, countIf(requestStatus = 'degraded') AS degraded, @@ -19,3 +19,4 @@ SQL > ORDER BY interval DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__http_uptime_7d__v1.pipe b/packages/tinybird/endpoints/endpoint__tcp_uptime_7d__v1.pipe similarity index 84% rename from packages/tinybird/pipes/endpoint__http_uptime_7d__v1.pipe rename to packages/tinybird/endpoints/endpoint__tcp_uptime_7d__v1.pipe index 1924c23cb..146ad31d7 100644 --- a/packages/tinybird/pipes/endpoint__http_uptime_7d__v1.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_uptime_7d__v1.pipe @@ -4,14 +4,14 @@ NODE endpoint SQL > % - SELECT + SELECT toStartOfInterval(time, INTERVAL {{ String(interval, '30', required=True) }} minute) AS interval, countIf(requestStatus = 'success') AS success, countIf(requestStatus = 'degraded') AS degraded, countIf(requestStatus = 'error') AS error - FROM mv__http_uptime_7d__v1 + FROM mv__tcp_uptime_7d__v1 WHERE - monitorId = {{ String(monitorId, '1', required=True) }} + monitorId = {{ String(monitorId, '4433', required=True) }} {% if fromDate %} AND time >= parseDateTimeBestEffortOrNull({{ String(fromDate) }}) {% end %} {% if toDate %} AND time <= parseDateTimeBestEffortOrNull({{ String(toDate) }}) {% end %} {% if regions %} AND region IN {{ Array(regions, 'String', 'ams,fra') }} {% end %} @@ -19,3 +19,4 @@ SQL > ORDER BY interval DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint__tcp_workspace_30d__v0.pipe b/packages/tinybird/endpoints/endpoint__tcp_workspace_30d__v0.pipe similarity index 94% rename from packages/tinybird/pipes/endpoint__tcp_workspace_30d__v0.pipe rename to packages/tinybird/endpoints/endpoint__tcp_workspace_30d__v0.pipe index bdf5a7b87..27568470c 100644 --- a/packages/tinybird/pipes/endpoint__tcp_workspace_30d__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_workspace_30d__v0.pipe @@ -13,3 +13,4 @@ SQL > ORDER BY day DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/endpoint_audit_log.pipe b/packages/tinybird/endpoints/endpoint_audit_log.pipe similarity index 91% rename from packages/tinybird/pipes/endpoint_audit_log.pipe rename to packages/tinybird/endpoints/endpoint_audit_log.pipe index 50d27b92f..4d42b16b2 100644 --- a/packages/tinybird/pipes/endpoint_audit_log.pipe +++ b/packages/tinybird/endpoints/endpoint_audit_log.pipe @@ -6,3 +6,4 @@ SQL > % SELECT * FROM audit_log__v0 WHERE id = {{ String(event_id, 1) }} ORDER BY timestamp DESC +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/external_status.pipe b/packages/tinybird/endpoints/endpoint_external_status.pipe similarity index 93% rename from packages/tinybird/pipes/external_status.pipe rename to packages/tinybird/endpoints/endpoint_external_status.pipe index 465029dc7..39f276fcb 100644 --- a/packages/tinybird/pipes/external_status.pipe +++ b/packages/tinybird/endpoints/endpoint_external_status.pipe @@ -10,3 +10,4 @@ SQL > LIMIT {{ Int16(limit, 10000) }} +TYPE ENDPOINT diff --git a/packages/tinybird/pipes/__ttl_45d_count_utc_get.pipe b/packages/tinybird/pipes/__ttl_45d_count_utc_get.pipe index 8270880ff..d91fd4a25 100644 --- a/packages/tinybird/pipes/__ttl_45d_count_utc_get.pipe +++ b/packages/tinybird/pipes/__ttl_45d_count_utc_get.pipe @@ -6,7 +6,7 @@ SQL > % SELECT time as day, countMerge(count) as count, countMerge(ok) as ok - FROM mv__http_status_45d + FROM mv__http_status_45d__v0 WHERE monitorId = {{ String(monitorId, '4') }} GROUP BY day @@ -17,5 +17,3 @@ SQL > TO toStartOfDay( date_sub(DAY, 45, now()) ) STEP INTERVAL -1 DAY - - diff --git a/packages/tinybird/pipes/aggregate__http_14d.pipe b/packages/tinybird/pipes/aggregate__http_14d.pipe deleted file mode 100644 index e546109c7..000000000 --- a/packages/tinybird/pipes/aggregate__http_14d.pipe +++ /dev/null @@ -1,28 +0,0 @@ -VERSION 0 - -TAGS http - -NODE aggregate -SQL > - - SELECT - toDateTime(fromUnixTimestamp64Milli(cronTimestamp)) AS time, - latency, - error, - region, - trigger, - statusCode, - timestamp, - cronTimestamp, - monitorId, - workspaceId - FROM ping_response__v8 - -TYPE materialized -DATASOURCE mv__http_14d__v0 -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(14)" - - diff --git a/packages/tinybird/pipes/aggregate__http_1d.pipe b/packages/tinybird/pipes/aggregate__http_1d.pipe deleted file mode 100644 index 3004300a6..000000000 --- a/packages/tinybird/pipes/aggregate__http_1d.pipe +++ /dev/null @@ -1,23 +0,0 @@ -TAGS "http" - -NODE aggregate -SQL > - - SELECT - toDateTime(fromUnixTimestamp64Milli(cronTimestamp)) AS time, - latency, - error, - requestStatus, - region, - trigger, - statusCode, - timestamp, - cronTimestamp, - monitorId, - workspaceId - FROM ping_response__v8 - -TYPE materialized -DATASOURCE mv__http_1d__v1 - - diff --git a/packages/tinybird/pipes/aggregate__http_30d.pipe b/packages/tinybird/pipes/aggregate__http_30d.pipe deleted file mode 100644 index 47e0f8fe2..000000000 --- a/packages/tinybird/pipes/aggregate__http_30d.pipe +++ /dev/null @@ -1,28 +0,0 @@ -VERSION 0 - -TAGS http - -NODE aggregate -SQL > - - SELECT - toDateTime(fromUnixTimestamp64Milli(cronTimestamp)) AS time, - latency, - error, - region, - trigger, - statusCode, - timestamp, - cronTimestamp, - monitorId, - workspaceId - FROM ping_response__v8 - -TYPE materialized -DATASOURCE mv__http_30d__v0 -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(30)" - - diff --git a/packages/tinybird/pipes/aggregate__http_7d.pipe b/packages/tinybird/pipes/aggregate__http_7d.pipe deleted file mode 100644 index d28c19f4d..000000000 --- a/packages/tinybird/pipes/aggregate__http_7d.pipe +++ /dev/null @@ -1,28 +0,0 @@ -VERSION 0 - -TAGS http - -NODE aggregate -SQL > - - SELECT - toDateTime(fromUnixTimestamp64Milli(cronTimestamp)) AS time, - latency, - error, - region, - trigger, - statusCode, - timestamp, - cronTimestamp, - monitorId, - workspaceId - FROM ping_response__v8 - -TYPE materialized -DATASOURCE mv__http_7d__v0 -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(7)" - - diff --git a/packages/tinybird/pipes/aggregate__http_full_30d.pipe b/packages/tinybird/pipes/aggregate__http_full_30d__v0.pipe similarity index 67% rename from packages/tinybird/pipes/aggregate__http_full_30d.pipe rename to packages/tinybird/pipes/aggregate__http_full_30d__v0.pipe index 02bb4c066..1402cda07 100644 --- a/packages/tinybird/pipes/aggregate__http_full_30d.pipe +++ b/packages/tinybird/pipes/aggregate__http_full_30d__v0.pipe @@ -1,9 +1,8 @@ -VERSION 0 DESCRIPTION > Stores all the data from the http_response table for the last 30 days, mainly used for accessing the data details. -TAGS http, full +TAGS "http, full" NODE aggregate SQL > @@ -15,9 +14,5 @@ SQL > TYPE materialized DATASOURCE mv__http_full_30d__v0 -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(30)" diff --git a/packages/tinybird/pipes/aggregate__http_status_14d.pipe b/packages/tinybird/pipes/aggregate__http_status_14d.pipe index 302131057..b9b48dbaa 100644 --- a/packages/tinybird/pipes/aggregate__http_status_14d.pipe +++ b/packages/tinybird/pipes/aggregate__http_status_14d.pipe @@ -22,6 +22,4 @@ SQL > monitorId TYPE materialized -DATASOURCE mv_http_status_14d__v0 - - +DATASOURCE mv__http_status_14d__v0 diff --git a/packages/tinybird/pipes/aggregate__http_status_45d.pipe b/packages/tinybird/pipes/aggregate__http_status_45d.pipe index 847160958..f502cc3e6 100644 --- a/packages/tinybird/pipes/aggregate__http_status_45d.pipe +++ b/packages/tinybird/pipes/aggregate__http_status_45d.pipe @@ -17,8 +17,3 @@ SQL > TYPE materialized DATASOURCE mv__http_status_45d__v0 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(45)" - diff --git a/packages/tinybird/pipes/aggregate__http_status_7d.pipe b/packages/tinybird/pipes/aggregate__http_status_7d.pipe index e40538586..f2e769959 100644 --- a/packages/tinybird/pipes/aggregate__http_status_7d.pipe +++ b/packages/tinybird/pipes/aggregate__http_status_7d.pipe @@ -17,8 +17,3 @@ SQL > TYPE materialized DATASOURCE mv__http_status_7d__v0 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(7)" - diff --git a/packages/tinybird/pipes/aggregate__tcp_14d.pipe b/packages/tinybird/pipes/aggregate__tcp_14d.pipe index 8ca2e6107..f1db16e55 100644 --- a/packages/tinybird/pipes/aggregate__tcp_14d.pipe +++ b/packages/tinybird/pipes/aggregate__tcp_14d.pipe @@ -19,9 +19,3 @@ SQL > TYPE materialized DATASOURCE mv__tcp_14d__v0 -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(14)" - - diff --git a/packages/tinybird/pipes/aggregate__tcp_1d.pipe b/packages/tinybird/pipes/aggregate__tcp_1d.pipe index d8ab99284..4d1356a57 100644 --- a/packages/tinybird/pipes/aggregate__tcp_1d.pipe +++ b/packages/tinybird/pipes/aggregate__tcp_1d.pipe @@ -19,9 +19,3 @@ SQL > TYPE materialized DATASOURCE mv__tcp_1d__v0 -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(1)" - - diff --git a/packages/tinybird/pipes/aggregate__tcp_30d.pipe b/packages/tinybird/pipes/aggregate__tcp_30d.pipe index b7dd911c6..deb04b5ed 100644 --- a/packages/tinybird/pipes/aggregate__tcp_30d.pipe +++ b/packages/tinybird/pipes/aggregate__tcp_30d.pipe @@ -19,9 +19,3 @@ SQL > TYPE materialized DATASOURCE mv__tcp_30d__v0 -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(30)" - - diff --git a/packages/tinybird/pipes/aggregate__tcp_7d.pipe b/packages/tinybird/pipes/aggregate__tcp_7d.pipe index 659d2be49..5270ab1c9 100644 --- a/packages/tinybird/pipes/aggregate__tcp_7d.pipe +++ b/packages/tinybird/pipes/aggregate__tcp_7d.pipe @@ -19,9 +19,3 @@ SQL > TYPE materialized DATASOURCE mv__tcp_7d__v0 -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(7)" - - diff --git a/packages/tinybird/pipes/aggregate__tcp_full_30d.pipe b/packages/tinybird/pipes/aggregate__tcp_full_30d__v0.pipe similarity index 52% rename from packages/tinybird/pipes/aggregate__tcp_full_30d.pipe rename to packages/tinybird/pipes/aggregate__tcp_full_30d__v0.pipe index c89b65410..e1fdc260f 100644 --- a/packages/tinybird/pipes/aggregate__tcp_full_30d.pipe +++ b/packages/tinybird/pipes/aggregate__tcp_full_30d__v0.pipe @@ -1,23 +1,16 @@ -VERSION 0 DESCRIPTION > Stores all the data from the http_response table for the last 30 days, mainly used for accessing the data details. -TAGS tcp, full +TAGS "tcp, full" NODE aggregate SQL > SELECT - toDateTime(fromUnixTimestamp64Milli(cronTimestamp)) AS time, + toDateTime(fromUnixTimestamp64Milli(tcp_response__v0.cronTimestamp)) AS time, * FROM tcp_response__v0 TYPE materialized DATASOURCE mv__tcp_full_30d__v0 -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(30)" - - diff --git a/packages/tinybird/pipes/aggregate__tcp_status_45d.pipe b/packages/tinybird/pipes/aggregate__tcp_status_45d.pipe index b6159db04..c3389940f 100644 --- a/packages/tinybird/pipes/aggregate__tcp_status_45d.pipe +++ b/packages/tinybird/pipes/aggregate__tcp_status_45d.pipe @@ -17,8 +17,3 @@ SQL > TYPE materialized DATASOURCE mv__tcp_status_45d__v0 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(45)" - diff --git a/packages/tinybird/pipes/aggregate__tcp_status_7d.pipe b/packages/tinybird/pipes/aggregate__tcp_status_7d.pipe index bd16b1867..777126266 100644 --- a/packages/tinybird/pipes/aggregate__tcp_status_7d.pipe +++ b/packages/tinybird/pipes/aggregate__tcp_status_7d.pipe @@ -17,8 +17,3 @@ SQL > TYPE materialized DATASOURCE mv__tcp_status_7d__v0 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "monitorId, time" -ENGINE_TTL "time + toIntervalDay(7)" - diff --git a/packages/tinybird/pipes/endpoint__http_metrics_by_interval_1d.pipe b/packages/tinybird/pipes/endpoint__http_metrics_by_interval_1d.pipe deleted file mode 100644 index f92d9fd60..000000000 --- a/packages/tinybird/pipes/endpoint__http_metrics_by_interval_1d.pipe +++ /dev/null @@ -1,27 +0,0 @@ -VERSION 0 - -TAGS http - -NODE endpoint -SQL > - - % - SELECT - region, - toStartOfInterval( - toDateTime(cronTimestamp / 1000), - INTERVAL {{ Int64(interval, 30) }} MINUTE -- use 2880 (2d) in case you want the 1d summary for the regions - ) as h, - toUnixTimestamp64Milli(toDateTime64(h, 3)) as timestamp, - round(quantile(0.50)(latency)) as p50Latency, - round(quantile(0.75)(latency)) as p75Latency, - round(quantile(0.90)(latency)) as p90Latency, - round(quantile(0.95)(latency)) as p95Latency, - round(quantile(0.99)(latency)) as p99Latency - FROM mv__http_1d__v0 - WHERE - monitorId = {{ String(monitorId, '1', required=True) }} - GROUP BY h, region - ORDER BY h DESC - - diff --git a/packages/tinybird/pipes/get_result_for_on_demand_check_http.pipe b/packages/tinybird/pipes/get_result_for_on_demand_check_http.pipe index 5f5971120..4053b4cfa 100644 --- a/packages/tinybird/pipes/get_result_for_on_demand_check_http.pipe +++ b/packages/tinybird/pipes/get_result_for_on_demand_check_http.pipe @@ -15,11 +15,9 @@ SQL > workspaceId, timing, cronTimestamp - FROM mv__http_full_30d + FROM mv__http_full_30d__v0 where - cronTimestamp = {{ String(timestamp, '1729871694536') }} and + cronTimestamp = {{ String(timestamp, '1729871694536') }} and monitorId = {{ String(monitorId, '2260') }} and url = {{ String(url , 'https://www.openstatus.dev/api/ping/edge') }} order by cronTimestamp desc - - diff --git a/packages/tinybird/pipes/public_status.pipe b/packages/tinybird/pipes/public_status.pipe index 588ec1b66..88f9c7cbe 100644 --- a/packages/tinybird/pipes/public_status.pipe +++ b/packages/tinybird/pipes/public_status.pipe @@ -11,7 +11,7 @@ SQL > cronTimestamp, count() AS count, count(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok - FROM ping_response_v5 + FROM ping_response__v8 WHERE monitorId = {{ String(monitorId, '1') }} {% if defined(url) %} AND url = {{ String(url) }} {% end %} @@ -20,5 +20,3 @@ SQL > GROUP BY cronTimestamp, monitorId ORDER BY cronTimestamp DESC LIMIT {{ Int16(limit, 5)}} - - diff --git a/packages/tinybird/pipes/response_details.pipe b/packages/tinybird/pipes/response_details.pipe index 63ca4949e..80d062cf4 100644 --- a/packages/tinybird/pipes/response_details.pipe +++ b/packages/tinybird/pipes/response_details.pipe @@ -5,11 +5,9 @@ SQL > % SELECT * - FROM ping_response__v7 + FROM ping_response__v8 WHERE monitorId = {{ String(monitorId, '1') }} {% if defined(url) %} AND url = {{ String(url) }} {% end %} AND cronTimestamp = {{ Int64(cronTimestamp, 1706467215188) }} AND region = {{ String(region, 'ams') }} - - diff --git a/packages/tinybird/pipes/response_graph.pipe b/packages/tinybird/pipes/response_graph.pipe index c48cd6188..34f9c0ff3 100644 --- a/packages/tinybird/pipes/response_graph.pipe +++ b/packages/tinybird/pipes/response_graph.pipe @@ -16,7 +16,7 @@ SQL > round(quantile(0.9)(latency)) as p90Latency, round(quantile(0.95)(latency)) as p95Latency, round(quantile(0.99)(latency)) as p99Latency - FROM ping_response__v7 + FROM ping_response__v8 WHERE monitorId = {{ String(monitorId, '1') }} {% if defined(url) %} AND url = {{ String(url) }} {% end %} @@ -24,5 +24,3 @@ SQL > {% if defined(toDate) %} AND timestamp <= {{ Int64(toDate) }} {% end %} GROUP BY h, region ORDER BY h DESC - - diff --git a/packages/tinybird/pipes/response_list.pipe b/packages/tinybird/pipes/response_list.pipe index cbc29493f..e194e4ecb 100644 --- a/packages/tinybird/pipes/response_list.pipe +++ b/packages/tinybird/pipes/response_list.pipe @@ -5,7 +5,7 @@ SQL > % SELECT latency, monitorId, region, statusCode, timestamp, url, workspaceId, cronTimestamp, message - FROM ping_response__v7 + FROM ping_response__v8 WHERE monitorId = {{ String(monitorId, 'openstatusPing') }} {% if defined(url) %} AND url = {{ String(url) }} {% end %} {% if defined(region) %} @@ -22,5 +22,3 @@ SQL > {% end %} ORDER BY timestamp DESC LIMIT {{Int32(limit, 100)}} - - diff --git a/packages/tinybird/pipes/response_time_metrics.pipe b/packages/tinybird/pipes/response_time_metrics.pipe deleted file mode 100644 index bb733b7ae..000000000 --- a/packages/tinybird/pipes/response_time_metrics.pipe +++ /dev/null @@ -1,55 +0,0 @@ -VERSION 0 - -NODE response_time_metrics__v0_0 -SQL > - - % - SELECT - round(avg(latency)) as avgLatency, - round(quantile(0.75)(latency)) as p75Latency, - round(quantile(0.9)(latency)) as p90Latency, - round(quantile(0.95)(latency)) as p95Latency, - round(quantile(0.99)(latency)) as p99Latency, - count() as count, - count(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok, - -- TODO: make it more efficient - ( - SELECT max(timestamp) - FROM materialized_view_ping_response_45d_ttl__v7 - WHERE - monitorId = {{ String(monitorId, '1') }} - {% if defined(url) %} AND url = {{ String(url) }} {% end %} - AND timestamp - >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL {{ Int32(interval, 24)}} HOUR, 3)) - ) as lastTimestamp, - -- for sorting - toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL {{ Int32(interval, 24)}} HOUR, 3)) as time - FROM materialized_view_ping_response_45d_ttl__v7 - WHERE - monitorId = {{ String(monitorId, '1') }} - {% if defined(url) %} AND url = {{ String(url) }} {% end %} - AND timestamp - >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL {{ Int32(interval, 24)}} HOUR, 3)) - UNION ALL - SELECT - round(avg(latency)) AS avgLatency, - round(quantile(0.75)(latency)) AS p75Latency, - round(quantile(0.9)(latency)) AS p90Latency, - round(quantile(0.95)(latency)) AS p95Latency, - round(quantile(0.99)(latency)) AS p99Latency, - count() as count, - count(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok, - NULL as lastTimestamp, -- no need to query the `lastTimestamp` as not relevant - -- for sorting - toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL {{ Int32(interval, 24)}}*2 HOUR, 3)) as time - FROM materialized_view_ping_response_45d_ttl__v7 - WHERE - monitorId = {{ String(monitorId, '1') }} - {% if defined(url) %} AND url = {{ String(url) }} {% end %} - AND timestamp - >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL {{ Int32(interval, 24)}}*2 HOUR, 3)) - AND timestamp - < toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL {{ Int32(interval, 24)}} HOUR, 3)) - - - diff --git a/packages/tinybird/pipes/response_time_metrics_by_region.pipe b/packages/tinybird/pipes/response_time_metrics_by_region.pipe deleted file mode 100644 index ed759c229..000000000 --- a/packages/tinybird/pipes/response_time_metrics_by_region.pipe +++ /dev/null @@ -1,24 +0,0 @@ -VERSION 0 - -NODE response_time_metrics_region__v0_0 -SQL > - - % - SELECT - region, - round(avg(latency)) as avgLatency, - round(quantile(0.75)(latency)) as p75Latency, - round(quantile(0.9)(latency)) as p90Latency, - round(quantile(0.95)(latency)) as p95Latency, - round(quantile(0.99)(latency)) as p99Latency - -- max(latency) as maxLatency, - -- min(latency) as minLatency - FROM materialized_view_ping_response_45d_ttl__v7 - WHERE - monitorId = {{ String(monitorId, '1') }} - {% if defined(url) %} AND url = {{ String(url) }} {% end %} - AND timestamp - >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL {{ Int32(interval, 24) }} HOUR, 3)) - GROUP BY region - - diff --git a/packages/tinybird/pipes/status_timezone.pipe b/packages/tinybird/pipes/status_timezone.pipe deleted file mode 100644 index 2b133e8ec..000000000 --- a/packages/tinybird/pipes/status_timezone.pipe +++ /dev/null @@ -1,49 +0,0 @@ -VERSION 1 - -DESCRIPTION > - TODO: descripe what it is for! - - -NODE group_by_cronTimestamp -SQL > - - % - SELECT - toDateTime(cronTimestamp / 1000, 'UTC') AS day, - toTimezone(day, {{ String(timezone, 'Europe/London') }}) as with_timezone, - toStartOfDay(with_timezone) as start_of_day, - -- latency, - statusCode - FROM ping_response__v7 - {% if defined(url) %} AND url = {{ String(url) }} {% end %} - WHERE monitorId = {{ String(monitorId, '1') }} - -- by default, we only query the last 45 days - AND cronTimestamp >= toUnixTimestamp64Milli( - toDateTime64(toStartOfDay(date_sub(DAY, 45, now())), 3) - ) - - -NODE group_by_day -SQL > - - % - SELECT - -- round(avg(latency)) as avgLatency, - -- round(quantile(0.75)(latency)) as p75Latency, - -- round(quantile(0.9)(latency)) as p90Latency, - -- round(quantile(0.95)(latency)) as p95Latency, - -- round(quantile(0.99)(latency)) as p99Latency, - start_of_day as day, - count() AS count, - count(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok - FROM group_by_cronTimestamp - GROUP BY start_of_day - ORDER BY start_of_day - WITH FILL - FROM - toStartOfDay(toTimezone(now(), {{ String(timezone, 'Europe/London') }})) - TO toStartOfDay( - toTimezone(date_sub(DAY, 46, now()), {{ String(timezone, 'Europe/London') }}) - ) STEP INTERVAL -1 DAY - - diff --git a/packages/tinybird/src/client.ts b/packages/tinybird/src/client.ts index 3c0aa54ed..f7998aab5 100644 --- a/packages/tinybird/src/client.ts +++ b/packages/tinybird/src/client.ts @@ -19,9 +19,14 @@ export class OSTinybird { if (process.env.NODE_ENV === "development") { this.tb = new NoopTinybird(); } else { - this.tb = new Client({ token }); + // Use local Tinybird container if available (Docker/self-hosted) + // https://www.tinybird.co/docs/api-reference + const tinybirdUrl = process.env.TINYBIRD_URL || "https://api.tinybird.co"; + this.tb = new Client({ + token, + baseUrl: tinybirdUrl, + }); } - // this.tb = new Client({ token }); } public get homeStats() { diff --git a/turbo.json b/turbo.json index f8dafeec0..ddc878fee 100644 --- a/turbo.json +++ b/turbo.json @@ -20,7 +20,10 @@ "UNKEY_API_ID", "UPSTASH_REDIS_REST_URL", "UPSTASH_REDIS_REST_TOKEN", - "EXTERNAL_API_URL" + "OPENPANEL_CLIENT_SECRET", + "NEXT_PUBLIC_OPENPANEL_CLIENT_ID", + "AUTH_SECRET", + "CRON_SECRET" ], "outputs": [".next/**", "!.next/cache/**", "dist/**"] },