hailey.at
+54
.github/workflows/api.yml
+54
.github/workflows/api.yml
···
1
+
name: Build and Push API
2
+
3
+
on:
4
+
push:
5
+
workflow_dispatch:
6
+
7
+
env:
8
+
REGISTRY: ghcr.io
9
+
IMAGE_NAME: ${{ github.repository }}/api
10
+
11
+
jobs:
12
+
build-and-push:
13
+
runs-on: ubuntu-latest
14
+
permissions:
15
+
contents: read
16
+
packages: write
17
+
18
+
steps:
19
+
- name: Checkout repository
20
+
uses: actions/checkout@v4
21
+
22
+
- name: Log in to the Container registry
23
+
uses: docker/login-action@v3
24
+
with:
25
+
registry: ${{ env.REGISTRY }}
26
+
username: ${{ github.actor }}
27
+
password: ${{ secrets.GITHUB_TOKEN }}
28
+
29
+
- name: Extract metadata (tags, labels) for Docker
30
+
id: meta
31
+
uses: docker/metadata-action@v5
32
+
with:
33
+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
34
+
tags: |
35
+
type=ref,event=branch
36
+
type=ref,event=pr
37
+
type=semver,pattern={{version}}
38
+
type=semver,pattern={{major}}.{{minor}}
39
+
type=semver,pattern={{major}}
40
+
type=sha,prefix={{branch}}-
41
+
42
+
- name: Set up Docker Buildx
43
+
uses: docker/setup-buildx-action@v3
44
+
45
+
- name: Build and push Docker image
46
+
uses: docker/build-push-action@v5
47
+
with:
48
+
context: .
49
+
file: ./cmd/api/Dockerfile
50
+
push: true
51
+
tags: ${{ steps.meta.outputs.tags }}
52
+
labels: ${{ steps.meta.outputs.labels }}
53
+
cache-from: type=gha
54
+
cache-to: type=gha,mode=max
+54
.github/workflows/database.yml
+54
.github/workflows/database.yml
···
1
+
name: Build and Push Database
2
+
3
+
on:
4
+
push:
5
+
workflow_dispatch:
6
+
7
+
env:
8
+
REGISTRY: ghcr.io
9
+
IMAGE_NAME: ${{ github.repository }}/database
10
+
11
+
jobs:
12
+
build-and-push:
13
+
runs-on: ubuntu-latest
14
+
permissions:
15
+
contents: read
16
+
packages: write
17
+
18
+
steps:
19
+
- name: Checkout repository
20
+
uses: actions/checkout@v4
21
+
22
+
- name: Log in to the Container registry
23
+
uses: docker/login-action@v3
24
+
with:
25
+
registry: ${{ env.REGISTRY }}
26
+
username: ${{ github.actor }}
27
+
password: ${{ secrets.GITHUB_TOKEN }}
28
+
29
+
- name: Extract metadata (tags, labels) for Docker
30
+
id: meta
31
+
uses: docker/metadata-action@v5
32
+
with:
33
+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
34
+
tags: |
35
+
type=ref,event=branch
36
+
type=ref,event=pr
37
+
type=semver,pattern={{version}}
38
+
type=semver,pattern={{major}}.{{minor}}
39
+
type=semver,pattern={{major}}
40
+
type=sha,prefix={{branch}}-
41
+
42
+
- name: Set up Docker Buildx
43
+
uses: docker/setup-buildx-action@v3
44
+
45
+
- name: Build and push Docker image
46
+
uses: docker/build-push-action@v5
47
+
with:
48
+
context: .
49
+
file: ./cmd/database/Dockerfile
50
+
push: true
51
+
tags: ${{ steps.meta.outputs.tags }}
52
+
labels: ${{ steps.meta.outputs.labels }}
53
+
cache-from: type=gha
54
+
cache-to: type=gha,mode=max
+54
.github/workflows/firehose.yml
+54
.github/workflows/firehose.yml
···
1
+
name: Build and Push Firehose
2
+
3
+
on:
4
+
push:
5
+
workflow_dispatch:
6
+
7
+
env:
8
+
REGISTRY: ghcr.io
9
+
IMAGE_NAME: ${{ github.repository }}/firehose
10
+
11
+
jobs:
12
+
build-and-push:
13
+
runs-on: ubuntu-latest
14
+
permissions:
15
+
contents: read
16
+
packages: write
17
+
18
+
steps:
19
+
- name: Checkout repository
20
+
uses: actions/checkout@v4
21
+
22
+
- name: Log in to the Container registry
23
+
uses: docker/login-action@v3
24
+
with:
25
+
registry: ${{ env.REGISTRY }}
26
+
username: ${{ github.actor }}
27
+
password: ${{ secrets.GITHUB_TOKEN }}
28
+
29
+
- name: Extract metadata (tags, labels) for Docker
30
+
id: meta
31
+
uses: docker/metadata-action@v5
32
+
with:
33
+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
34
+
tags: |
35
+
type=ref,event=branch
36
+
type=ref,event=pr
37
+
type=semver,pattern={{version}}
38
+
type=semver,pattern={{major}}.{{minor}}
39
+
type=semver,pattern={{major}}
40
+
type=sha,prefix={{branch}}-
41
+
42
+
- name: Set up Docker Buildx
43
+
uses: docker/setup-buildx-action@v3
44
+
45
+
- name: Build and push Docker image
46
+
uses: docker/build-push-action@v5
47
+
with:
48
+
context: .
49
+
file: ./cmd/bus/firehose/Dockerfile
50
+
push: true
51
+
tags: ${{ steps.meta.outputs.tags }}
52
+
labels: ${{ steps.meta.outputs.labels }}
53
+
cache-from: type=gha
54
+
cache-to: type=gha,mode=max
+54
.github/workflows/indexer.yml
+54
.github/workflows/indexer.yml
···
1
+
name: Build and Push Indexer
2
+
3
+
on:
4
+
push:
5
+
workflow_dispatch:
6
+
7
+
env:
8
+
REGISTRY: ghcr.io
9
+
IMAGE_NAME: ${{ github.repository }}/indexer
10
+
11
+
jobs:
12
+
build-and-push:
13
+
runs-on: ubuntu-latest
14
+
permissions:
15
+
contents: read
16
+
packages: write
17
+
18
+
steps:
19
+
- name: Checkout repository
20
+
uses: actions/checkout@v4
21
+
22
+
- name: Log in to the Container registry
23
+
uses: docker/login-action@v3
24
+
with:
25
+
registry: ${{ env.REGISTRY }}
26
+
username: ${{ github.actor }}
27
+
password: ${{ secrets.GITHUB_TOKEN }}
28
+
29
+
- name: Extract metadata (tags, labels) for Docker
30
+
id: meta
31
+
uses: docker/metadata-action@v5
32
+
with:
33
+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
34
+
tags: |
35
+
type=ref,event=branch
36
+
type=ref,event=pr
37
+
type=semver,pattern={{version}}
38
+
type=semver,pattern={{major}}.{{minor}}
39
+
type=semver,pattern={{major}}
40
+
type=sha,prefix={{branch}}-
41
+
42
+
- name: Set up Docker Buildx
43
+
uses: docker/setup-buildx-action@v3
44
+
45
+
- name: Build and push Docker image
46
+
uses: docker/build-push-action@v5
47
+
with:
48
+
context: .
49
+
file: ./cmd/indexer/Dockerfile
50
+
push: true
51
+
tags: ${{ steps.meta.outputs.tags }}
52
+
labels: ${{ steps.meta.outputs.labels }}
53
+
cache-from: type=gha
54
+
cache-to: type=gha,mode=max
+28
cmd/api/Dockerfile
+28
cmd/api/Dockerfile
···
1
+
FROM golang:1.25-alpine AS builder
2
+
3
+
WORKDIR /app
4
+
5
+
# Copy go mod files
6
+
COPY go.mod go.sum ./
7
+
RUN go mod download
8
+
9
+
# Copy source code
10
+
COPY . .
11
+
12
+
# Build the binary
13
+
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o api ./cmd/api
14
+
15
+
FROM alpine:latest
16
+
17
+
RUN apk --no-cache add ca-certificates
18
+
19
+
WORKDIR /root/
20
+
21
+
# Copy binary from builder
22
+
COPY --from=builder /app/api .
23
+
24
+
# Expose port
25
+
EXPOSE 8080
26
+
27
+
# Run the binary
28
+
CMD ["./api"]
+25
cmd/bus/firehose/Dockerfile
+25
cmd/bus/firehose/Dockerfile
···
1
+
FROM golang:1.25-alpine AS builder
2
+
3
+
WORKDIR /app
4
+
5
+
# Copy go mod files
6
+
COPY go.mod go.sum ./
7
+
RUN go mod download
8
+
9
+
# Copy source code
10
+
COPY . .
11
+
12
+
# Build the binary
13
+
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o firehose ./cmd/bus/firehose
14
+
15
+
FROM alpine:latest
16
+
17
+
RUN apk --no-cache add ca-certificates
18
+
19
+
WORKDIR /root/
20
+
21
+
# Copy binary from builder
22
+
COPY --from=builder /app/firehose .
23
+
24
+
# Run the binary
25
+
CMD ["./firehose"]
+28
cmd/database/Dockerfile
+28
cmd/database/Dockerfile
···
1
+
FROM golang:1.25-alpine AS builder
2
+
3
+
WORKDIR /app
4
+
5
+
# Copy go mod files
6
+
COPY go.mod go.sum ./
7
+
RUN go mod download
8
+
9
+
# Copy source code
10
+
COPY . .
11
+
12
+
# Build the binary
13
+
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o database ./cmd/database
14
+
15
+
FROM alpine:latest
16
+
17
+
RUN apk --no-cache add ca-certificates
18
+
19
+
WORKDIR /root/
20
+
21
+
# Copy binary from builder
22
+
COPY --from=builder /app/database .
23
+
24
+
# Expose port
25
+
EXPOSE 9090
26
+
27
+
# Run the binary
28
+
CMD ["./database"]
+25
cmd/indexer/Dockerfile
+25
cmd/indexer/Dockerfile
···
1
+
FROM golang:1.25-alpine AS builder
2
+
3
+
WORKDIR /app
4
+
5
+
# Copy go mod files
6
+
COPY go.mod go.sum ./
7
+
RUN go mod download
8
+
9
+
# Copy source code
10
+
COPY . .
11
+
12
+
# Build the binary
13
+
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o indexer ./cmd/indexer
14
+
15
+
FROM alpine:latest
16
+
17
+
RUN apk --no-cache add ca-certificates
18
+
19
+
WORKDIR /root/
20
+
21
+
# Copy binary from builder
22
+
COPY --from=builder /app/indexer .
23
+
24
+
# Run the binary
25
+
CMD ["./indexer"]
+192
docker-compose-staging.yaml
+192
docker-compose-staging.yaml
···
1
+
services:
2
+
zookeeper:
3
+
image: confluentinc/cp-zookeeper:7.6.0
4
+
hostname: zookeeper
5
+
container_name: zookeeper
6
+
ports:
7
+
- "2181:2181"
8
+
environment:
9
+
ZOOKEEPER_CLIENT_PORT: 2181
10
+
ZOOKEEPER_TICK_TIME: 2000
11
+
volumes:
12
+
- zookeeper-data:/var/lib/zookeeper/data
13
+
- zookeeper-logs:/var/lib/zookeeper/log
14
+
15
+
kafka1:
16
+
image: confluentinc/cp-kafka:7.6.0
17
+
hostname: kafka1
18
+
container_name: kafka1
19
+
depends_on:
20
+
- zookeeper
21
+
ports:
22
+
- "9092:9092"
23
+
- "9101:9101"
24
+
environment:
25
+
KAFKA_BROKER_ID: 1
26
+
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
27
+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
28
+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:29092,PLAINTEXT_HOST://localhost:9092
29
+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
30
+
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
31
+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
32
+
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
33
+
KAFKA_JMX_PORT: 9101
34
+
KAFKA_JMX_HOSTNAME: localhost
35
+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
36
+
volumes:
37
+
- kafka1-data:/var/lib/kafka/data
38
+
39
+
kafka2:
40
+
image: confluentinc/cp-kafka:7.6.0
41
+
hostname: kafka2
42
+
container_name: kafka2
43
+
depends_on:
44
+
- zookeeper
45
+
ports:
46
+
- "9093:9093"
47
+
- "9102:9102"
48
+
environment:
49
+
KAFKA_BROKER_ID: 2
50
+
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
51
+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
52
+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:29093,PLAINTEXT_HOST://localhost:9093
53
+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
54
+
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
55
+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
56
+
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
57
+
KAFKA_JMX_PORT: 9102
58
+
KAFKA_JMX_HOSTNAME: localhost
59
+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
60
+
volumes:
61
+
- kafka2-data:/var/lib/kafka/data
62
+
63
+
kafka3:
64
+
image: confluentinc/cp-kafka:7.6.0
65
+
hostname: kafka3
66
+
container_name: kafka3
67
+
depends_on:
68
+
- zookeeper
69
+
ports:
70
+
- "9094:9094"
71
+
- "9103:9103"
72
+
environment:
73
+
KAFKA_BROKER_ID: 3
74
+
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
75
+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
76
+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:29094,PLAINTEXT_HOST://localhost:9094
77
+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
78
+
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
79
+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
80
+
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
81
+
KAFKA_JMX_PORT: 9103
82
+
KAFKA_JMX_HOSTNAME: localhost
83
+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
84
+
volumes:
85
+
- kafka3-data:/var/lib/kafka/data
86
+
87
+
cassandra:
88
+
image: cassandra:5.0
89
+
hostname: cassandra
90
+
container_name: cassandra
91
+
ports:
92
+
- "9042:9042" # CQL native transport port
93
+
- "7000:7000" # Inter-node cluster communication
94
+
- "7199:7199" # JMX monitoring port
95
+
environment:
96
+
CASSANDRA_CLUSTER_NAME: 'MyCluster'
97
+
CASSANDRA_DC: 'dc1'
98
+
CASSANDRA_RACK: 'rack1'
99
+
CASSANDRA_ENDPOINT_SNITCH: 'GossipingPropertyFileSnitch'
100
+
CASSANDRA_NUM_TOKENS: 256
101
+
# Reduce memory usage for local development
102
+
MAX_HEAP_SIZE: '512M'
103
+
HEAP_NEWSIZE: '128M'
104
+
# Override JVM opts to limit direct memory to fit in container
105
+
JVM_OPTS: '-Xms512M -Xmx512M -XX:MaxDirectMemorySize=256M'
106
+
volumes:
107
+
- cassandra-data:/var/lib/cassandra
108
+
# Limit container memory (increased to accommodate heap + direct memory + overhead)
109
+
mem_limit: 2g
110
+
memswap_limit: 2g
111
+
# Disable swap for better performance
112
+
mem_swappiness: 0
113
+
# Allow memory locking (reduces warnings)
114
+
ulimits:
115
+
memlock: -1
116
+
nofile:
117
+
soft: 65536
118
+
hard: 65536
119
+
healthcheck:
120
+
test: ["CMD-SHELL", "cqlsh -e 'describe cluster'"]
121
+
interval: 30s
122
+
timeout: 10s
123
+
retries: 5
124
+
125
+
database:
126
+
build:
127
+
context: .
128
+
dockerfile: ./cmd/database/Dockerfile
129
+
container_name: vylet-database
130
+
depends_on:
131
+
cassandra:
132
+
condition: service_healthy
133
+
ports:
134
+
- "9090:9090"
135
+
environment:
136
+
VYLET_DATABASE_LISTEN_ADDR: ":9090"
137
+
command: ["./database", "--cassandra-addrs", "cassandra", "--cassandra-keyspace", "vylet"]
138
+
restart: unless-stopped
139
+
140
+
api:
141
+
build:
142
+
context: .
143
+
dockerfile: ./cmd/api/Dockerfile
144
+
container_name: vylet-api
145
+
depends_on:
146
+
- database
147
+
ports:
148
+
- "8080:8080"
149
+
environment:
150
+
VYLET_API_LISTEN_ADDR: ":8080"
151
+
VYLET_API_DB_HOST: "database:9090"
152
+
restart: unless-stopped
153
+
154
+
firehose:
155
+
build:
156
+
context: .
157
+
dockerfile: ./cmd/bus/firehose/Dockerfile
158
+
container_name: vylet-firehose
159
+
depends_on:
160
+
- kafka1
161
+
- kafka2
162
+
- kafka3
163
+
environment:
164
+
KAFKA_FIREHOSE_WEBSOCKET_HOST: "wss://bsky.network"
165
+
KAFKA_FIREHOSE_BOOTSTRAP_SERVERS: "kafka1:29092,kafka2:29093,kafka3:29094"
166
+
KAFKA_FIREHOSE_OUTPUT_TOPIC: "firehose-events-prod"
167
+
restart: unless-stopped
168
+
169
+
indexer:
170
+
build:
171
+
context: .
172
+
dockerfile: ./cmd/indexer/Dockerfile
173
+
container_name: vylet-indexer
174
+
depends_on:
175
+
- kafka1
176
+
- kafka2
177
+
- kafka3
178
+
- database
179
+
environment:
180
+
VYLET_DATABASE_HOST: "database:9090"
181
+
VYLET_BOOTSTRAP_SERVERS: "kafka1:29092,kafka2:29093,kafka3:29094"
182
+
VYLET_INDEXER_INPUT_TOPIC: "firehose-events-prod"
183
+
VYLET_INDEXER_CONSUMER_GROUP: "vylet-indexer-staging"
184
+
restart: unless-stopped
185
+
186
+
volumes:
187
+
zookeeper-data:
188
+
zookeeper-logs:
189
+
kafka1-data:
190
+
kafka2-data:
191
+
kafka3-data:
192
+
cassandra-data: