using System;
using System.Buffers;
using System.Collections.Generic;
using System.Net.WebSockets;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using ZstdSharp;
namespace CarpaNet.Jetstream;
///
/// Client for consuming the Jetstream event stream (JSON over WebSocket).
/// Jetstream provides a lightweight alternative to the full ATProtocol firehose,
/// delivering decoded records as JSON instead of raw CBOR/CAR blocks.
///
public sealed class JetstreamClient : IDisposable
{
private readonly Uri _baseUri;
private readonly JsonSerializerOptions _jsonOptions;
private readonly Decompressor? _decompressor;
private ClientWebSocket? _webSocket;
private bool _disposed;
///
/// Default buffer size for receiving WebSocket frames.
///
public int BufferSize { get; set; } = 512 * 1024; // 512KB default
///
/// Creates a new JetstreamClient.
///
/// The Jetstream instance URI (e.g., https://jetstream1.us-east.bsky.network).
/// Optional zstd dictionary bytes for decompressing binary frames.
/// Required when using = true.
/// The dictionary can be obtained from the Jetstream repository at
/// https://github.com/bluesky-social/jetstream/raw/main/pkg/models/zstd_dictionary
public JetstreamClient(Uri baseUri, byte[]? zstdDictionary = null)
: this(baseUri, JetstreamJsonContext.Default.Options, zstdDictionary)
{
}
///
/// Creates a new JetstreamClient with custom JSON options.
///
/// The Jetstream instance URI.
/// Custom JSON serializer options.
/// Optional zstd dictionary bytes for decompressing binary frames.
public JetstreamClient(Uri baseUri, JsonSerializerOptions jsonOptions, byte[]? zstdDictionary = null)
{
_baseUri = baseUri ?? throw new ArgumentNullException(nameof(baseUri));
_jsonOptions = jsonOptions ?? throw new ArgumentNullException(nameof(jsonOptions));
if (zstdDictionary != null)
{
_decompressor = new Decompressor();
_decompressor.LoadDictionary(zstdDictionary);
}
}
///
/// Subscribes to the Jetstream event stream and yields deserialized events.
///
/// Optional subscription options for filtering and cursor.
/// Cancellation token.
/// An async enumerable of Jetstream events.
public async IAsyncEnumerable SubscribeAsync(
JetstreamSubscribeOptions? options = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ThrowIfDisposed();
var uri = BuildWebSocketUri(options);
_webSocket = new ClientWebSocket();
try
{
await _webSocket.ConnectAsync(uri, cancellationToken).ConfigureAwait(false);
var buffer = ArrayPool.Shared.Rent(BufferSize);
try
{
while (!cancellationToken.IsCancellationRequested &&
_webSocket.State == WebSocketState.Open)
{
var result = await ReceiveFullMessageAsync(buffer, cancellationToken).ConfigureAwait(false);
if (result.MessageType == WebSocketMessageType.Close)
{
yield break;
}
JetstreamEvent? evt = null;
if (result.MessageType == WebSocketMessageType.Text)
{
var json = buffer.AsSpan(0, result.Count);
evt = JsonSerializer.Deserialize(json, JetstreamJsonContext.Default.JetstreamEvent);
}
else if (result.MessageType == WebSocketMessageType.Binary)
{
if (_decompressor == null)
{
throw new InvalidOperationException(
"Received zstd-compressed binary frame but no zstd dictionary was provided. " +
"Pass the Jetstream zstd dictionary to the JetstreamClient constructor to enable compression support.");
}
var compressed = buffer.AsSpan(0, result.Count);
var decompressed = _decompressor.Unwrap(compressed);
evt = JsonSerializer.Deserialize(decompressed, JetstreamJsonContext.Default.JetstreamEvent);
}
if (evt != null)
{
yield return evt;
}
}
}
finally
{
ArrayPool.Shared.Return(buffer);
}
}
finally
{
if (_webSocket.State == WebSocketState.Open)
{
try
{
await _webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closing", CancellationToken.None)
.ConfigureAwait(false);
}
catch
{
// Ignore close errors
}
}
_webSocket.Dispose();
_webSocket = null;
}
}
///
/// Subscribes to the Jetstream event stream and yields raw message bytes.
/// Use this when you need to handle decompression or custom deserialization.
///
/// Optional subscription options for filtering and cursor.
/// Cancellation token.
/// An async enumerable of (message type, message bytes) tuples.
public async IAsyncEnumerable<(WebSocketMessageType MessageType, ReadOnlyMemory Data)> SubscribeRawAsync(
JetstreamSubscribeOptions? options = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ThrowIfDisposed();
var uri = BuildWebSocketUri(options);
_webSocket = new ClientWebSocket();
try
{
await _webSocket.ConnectAsync(uri, cancellationToken).ConfigureAwait(false);
var buffer = ArrayPool.Shared.Rent(BufferSize);
try
{
while (!cancellationToken.IsCancellationRequested &&
_webSocket.State == WebSocketState.Open)
{
var result = await ReceiveFullMessageAsync(buffer, cancellationToken).ConfigureAwait(false);
if (result.MessageType == WebSocketMessageType.Close)
{
yield break;
}
var data = new byte[result.Count];
Buffer.BlockCopy(buffer, 0, data, 0, result.Count);
yield return (result.MessageType, data);
}
}
finally
{
ArrayPool.Shared.Return(buffer);
}
}
finally
{
if (_webSocket.State == WebSocketState.Open)
{
try
{
await _webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closing", CancellationToken.None)
.ConfigureAwait(false);
}
catch
{
// Ignore close errors
}
}
_webSocket.Dispose();
_webSocket = null;
}
}
///
/// Sends an options_update message to dynamically change subscription filters.
/// Only valid after connecting with = true
/// or to update filters on an active connection.
///
/// The options update to send.
/// Cancellation token.
public async Task SendOptionsUpdateAsync(JetstreamOptionsUpdate update, CancellationToken cancellationToken = default)
{
ThrowIfDisposed();
if (_webSocket == null || _webSocket.State != WebSocketState.Open)
{
throw new InvalidOperationException("WebSocket is not connected");
}
var json = JsonSerializer.SerializeToUtf8Bytes(update, JetstreamJsonContext.Default.JetstreamOptionsUpdate);
await _webSocket.SendAsync(
new ArraySegment(json),
WebSocketMessageType.Text,
true,
cancellationToken).ConfigureAwait(false);
}
private async Task ReceiveFullMessageAsync(byte[] buffer, CancellationToken cancellationToken)
{
var totalBytes = 0;
WebSocketReceiveResult result;
do
{
var segment = new ArraySegment(buffer, totalBytes, buffer.Length - totalBytes);
result = await _webSocket!.ReceiveAsync(segment, cancellationToken).ConfigureAwait(false);
totalBytes += result.Count;
if (totalBytes >= buffer.Length && !result.EndOfMessage)
{
throw new InvalidOperationException("Jetstream message too large for buffer");
}
} while (!result.EndOfMessage);
return new WebSocketReceiveResult(totalBytes, result.MessageType, true, result.CloseStatus, result.CloseStatusDescription);
}
private Uri BuildWebSocketUri(JetstreamSubscribeOptions? options)
{
var uriBuilder = new UriBuilder(_baseUri)
{
Path = "/subscribe"
};
if (options != null)
{
var queryParams = new List();
foreach (var param in options.ToQueryParameters())
{
queryParams.Add($"{Uri.EscapeDataString(param.Key)}={Uri.EscapeDataString(param.Value)}");
}
if (queryParams.Count > 0)
{
uriBuilder.Query = string.Join("&", queryParams);
}
}
// Ensure WebSocket scheme
if (uriBuilder.Scheme == "http")
{
uriBuilder.Scheme = "ws";
}
else if (uriBuilder.Scheme == "https")
{
uriBuilder.Scheme = "wss";
}
return uriBuilder.Uri;
}
private void ThrowIfDisposed()
{
#if NET8_0_OR_GREATER
ObjectDisposedException.ThrowIf(_disposed, this);
#else
if (_disposed)
{
throw new ObjectDisposedException(GetType().FullName);
}
#endif
}
///
/// Disposes the client and closes any open connections.
///
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
_decompressor?.Dispose();
_webSocket?.Dispose();
}
}