Elixir ATProtocol firehose & subscription listener

refactor: simplify structure. Remove ConsumerGroup and make Consumer a simple behaviour

ovyerus.com 8ba72831 edbdd5b4

verified
+3 -4
examples/record_consumer.ex
··· 21 21 Supervisor.start_link(__MODULE__, arg, name: __MODULE__) 22 22 end 23 23 24 - @immpl true 25 - def init(_arg) do 24 + @impl true 25 + def init(_) do 26 26 children = [ 27 - Drinkup, 28 - ExampleRecordConsumer 27 + {Drinkup, %{module: ExampleRecordConsumer}} 29 28 ] 30 29 31 30 Supervisor.init(children, strategy: :one_for_one)
+1 -50
lib/consumer.ex
··· 3 3 An unopinionated consumer of the Firehose. Will receive all events, not just commits. 4 4 """ 5 5 6 - alias Drinkup.{ConsumerGroup, Event} 6 + alias Drinkup.Event 7 7 8 8 @callback handle_event(Event.t()) :: any() 9 - 10 - defmacro __using__(_opts) do 11 - quote location: :keep do 12 - use GenServer 13 - require Logger 14 - 15 - @behaviour Drinkup.Consumer 16 - 17 - def child_spec(opts) do 18 - %{ 19 - id: __MODULE__, 20 - start: {__MODULE__, :start_link, [opts]}, 21 - type: :worker, 22 - restart: :permanent, 23 - max_restarts: 0, 24 - shutdown: 500 25 - } 26 - end 27 - 28 - def start_link(opts) do 29 - GenServer.start_link(__MODULE__, [], opts) 30 - end 31 - 32 - @impl GenServer 33 - def init(_) do 34 - ConsumerGroup.join() 35 - {:ok, nil} 36 - end 37 - 38 - @impl GenServer 39 - def handle_info({:event, event}, state) do 40 - {:ok, _pid} = 41 - Task.start(fn -> 42 - try do 43 - __MODULE__.handle_event(event) 44 - rescue 45 - e -> 46 - Logger.error( 47 - "Error in event handler: #{Exception.format(:error, e, __STACKTRACE__)}" 48 - ) 49 - end 50 - end) 51 - 52 - {:noreply, state} 53 - end 54 - 55 - defoverridable GenServer 56 - end 57 - end 58 9 end
-39
lib/consumer_group.ex
··· 1 - defmodule Drinkup.ConsumerGroup do 2 - @moduledoc """ 3 - Register consumers and dispatch events to them. 4 - """ 5 - 6 - alias Drinkup.Event 7 - 8 - @scope __MODULE__ 9 - @group :consumers 10 - 11 - def start_link(_) do 12 - :pg.start_link(@scope) 13 - end 14 - 15 - def child_spec(opts) do 16 - %{ 17 - id: __MODULE__, 18 - start: {__MODULE__, :start_link, [opts]}, 19 - type: :worker, 20 - restart: :permanent, 21 - shutdown: 500 22 - } 23 - end 24 - 25 - @spec join() :: :ok 26 - def join(), do: join(self()) 27 - 28 - @spec join(pid()) :: :ok 29 - def join(pid), do: :pg.join(@scope, @group, pid) 30 - 31 - @spec dispatch(Event.t()) :: :ok 32 - def dispatch(event) do 33 - @scope 34 - |> :pg.get_members(@group) 35 - |> Enum.each(&send(&1, {:event, event})) 36 - end 37 - 38 - # TODO: read `:pg` docs on what `monitor` is used fo 39 - end
+12 -5
lib/drinkup.ex
··· 1 1 defmodule Drinkup do 2 2 use Supervisor 3 3 4 - def start_link(arg \\ []) do 5 - Supervisor.start_link(__MODULE__, arg, name: __MODULE__) 4 + @type options() :: %{ 5 + required(:consumer) => module(), 6 + optional(:host) => String.t(), 7 + optional(:cursor) => pos_integer() 8 + } 9 + 10 + @spec start_link(options()) :: Supervisor.on_start() 11 + def start_link(options) do 12 + Supervisor.start_link(__MODULE__, options, name: __MODULE__) 6 13 end 7 14 8 - def init(_) do 15 + def init(options) do 9 16 children = [ 10 - Drinkup.ConsumerGroup, 11 - Drinkup.Socket 17 + {Task.Supervisor, name: Drinkup.TaskSupervisor}, 18 + {Drinkup.Socket, options} 12 19 ] 13 20 14 21 Supervisor.init(children, strategy: :one_for_one)
+16
lib/event.ex
··· 1 1 defmodule Drinkup.Event do 2 + require Logger 2 3 alias Drinkup.Event 3 4 4 5 @type t() :: ··· 21 22 def valid_seq?(last_seq, nil) when is_integer(last_seq), do: true 22 23 def valid_seq?(last_seq, seq) when is_integer(last_seq) and is_integer(seq), do: seq > last_seq 23 24 def valid_seq?(_last_seq, _seq), do: false 25 + 26 + @spec dispatch(module(), t()) :: :ok 27 + def dispatch(consumer, message) do 28 + {:ok, _pid} = 29 + Task.Supervisor.start_child(Drinkup.TaskSupervisor, fn -> 30 + try do 31 + consumer.handle_event(message) 32 + rescue 33 + e -> 34 + Logger.error("Error in event handler: #{Exception.format(:error, e, __STACKTRACE__)}") 35 + end 36 + end) 37 + 38 + :ok 39 + end 24 40 end
+1 -1
lib/record_consumer.ex
··· 11 11 {collections, _opts} = Keyword.pop(opts, :collections, []) 12 12 13 13 quote location: :keep do 14 - use Drinkup.Consumer 14 + @behaviour Drinkup.Consumer 15 15 @behaviour Drinkup.RecordConsumer 16 16 17 17 def handle_event(%Drinkup.Event.Commit{} = event) do
+11 -13
lib/socket.ex
··· 4 4 """ 5 5 6 6 require Logger 7 - alias Drinkup.{ConsumerGroup, Event} 7 + alias Drinkup.Event 8 8 9 9 @behaviour :gen_statem 10 10 @default_host "https://bsky.network" ··· 15 15 @op_regular 1 16 16 @op_error -1 17 17 18 - defstruct [:host, :seq, :conn, :stream] 18 + defstruct [:options, :seq, :conn, :stream] 19 19 20 20 @impl true 21 21 def callback_mode, do: [:state_functions, :state_enter] ··· 30 30 } 31 31 end 32 32 33 - def start_link(opts \\ [], statem_opts) do 34 - opts = Keyword.validate!(opts, host: @default_host) 35 - host = Keyword.get(opts, :host) 36 - cursor = Keyword.get(opts, :cursor) 33 + def start_link(%{consumer: _} = options, statem_opts) do 34 + options = Map.merge(%{host: @default_host, cursor: nil}, options) 37 35 38 - :gen_statem.start_link(__MODULE__, {host, cursor}, statem_opts) 36 + :gen_statem.start_link(__MODULE__, options, statem_opts) 39 37 end 40 38 41 39 @impl true 42 - def init({host, cursor}) do 43 - data = %__MODULE__{host: host, seq: cursor} 40 + def init(%{cursor: seq} = options) do 41 + data = %__MODULE__{seq: seq, options: options} 44 42 {:ok, :disconnected, data, [{:next_event, :internal, :connect}]} 45 43 end 46 44 ··· 54 52 {:next_state, :connecting_http, data} 55 53 end 56 54 57 - def connecting_http(:enter, _from, data) do 55 + def connecting_http(:enter, _from, %{options: options} = data) do 58 56 Logger.debug("Connecting to http") 59 57 60 - %{host: host, port: port} = URI.new!(data.host) 58 + %{host: host, port: port} = URI.new!(options.host) 61 59 62 60 {:ok, conn} = 63 61 :gun.open(:binary.bin_to_list(host), port, %{ ··· 107 105 :keep_state_and_data 108 106 end 109 107 110 - def connected(:info, {:gun_ws, conn, stream, {:binary, frame}}, data) do 108 + def connected(:info, {:gun_ws, conn, stream, {:binary, frame}}, %{options: options} = data) do 111 109 # TODO: let clients specify a handler for raw* (*decoded) packets to support any atproto subscription 112 110 # Will also need support for JSON frames 113 111 with {:ok, header, next} <- CAR.DagCbor.decode(frame), ··· 123 121 Logger.warning("Received unrecognised event from firehose: #{inspect({type, payload})}") 124 122 125 123 message -> 126 - ConsumerGroup.dispatch(message) 124 + Event.dispatch(options.consumer, message) 127 125 end 128 126 129 127 {:keep_state, data}