use crate::storage::{ queue::{DequeueQuote, EnqueueQuotes}, source::SourceQuotes, }; use crate::data::Quote; use crate::data::{Handle, Message}; mod rng { use rand::SeedableRng; pub struct PrngState { rng: rand::rngs::SmallRng, } impl PrngState { pub fn from_thread_rng() -> Self { Self { rng: rand::rngs::SmallRng::from_rng(&mut rand::rng()), } } pub fn from_seed(seed: u64) -> Self { Self { rng: rand::rngs::SmallRng::seed_from_u64(seed), } } pub fn shuffle_slice(&mut self, slice: &mut [T]) { use rand::seq::SliceRandom; slice.shuffle(&mut self.rng); } } mod test { #[tokio::test] async fn shuffle_slice() { use super::*; let mut data = vec![1, 2, 3, 4]; let mut rng = PrngState::from_thread_rng(); rng.shuffle_slice(&mut data); println!("{:?}", data); } } } pub mod source { use super::*; // TODO: Should the quote source filters be // generic over the exact manager implementation being used? /// Message to request that a SourceManager source its quotes once again. pub struct SourceQuotes; pub type SourceReply = Result, ()>; impl Message for SourceQuotes { type Reply = SourceReply; } /// Subtrait of Actor which specifically /// denotes actors that can handle all relevant source messages. pub trait SourceManager: Handle {} impl SourceManager for T where T: Handle {} /// Implementation of [SourceManager] which sources quotes from a Vec /// that it holds in memory, without accessing external services. /// Its main purpose is to be used for testing. pub struct MemorySourceManager { quotes: Vec, } impl MemorySourceManager { pub fn new(quotes: impl IntoIterator>) -> Self { Self { quotes: quotes.into_iter().map(Into::into).collect(), } } } #[async_trait::async_trait(?Send)] impl Handle for MemorySourceManager { async fn handle(&mut self, _msg: SourceQuotes) -> Result, ()> { // We just clone the quotes we've been holding onto since startup Ok(self.quotes.clone()) } } /// Uses a [QuoteFilter] to source quotes from the local filesystem /// at the beginning of each cycle. pub struct FsFilterSourceManager { filter: QuoteFilter, } impl FsFilterSourceManager { pub fn new(filter: QuoteFilter) -> Self { Self { filter } } } impl Default for FsFilterSourceManager { fn default() -> Self { Self { filter: QuoteFilter { content: r".*".to_string(), // TODO: Maybe make this a compile-time constant for debugging path: "test/**/*.txt".to_string(), _dates: vec![], }, } } } #[async_trait::async_trait(?Send)] impl Handle for FsFilterSourceManager { async fn handle(&mut self, _msg: SourceQuotes) -> Result, ()> { self.filter.read_files() } } #[derive(Clone, Debug)] pub struct QuoteFilter { /// "glob-like" pattern of files containing quotes to be read. path: String, /// Regular expression to further filter the files matched by [path] /// based on their *contents*. content: String, /// Currently unused. Intended for future functionality. _dates: Vec, } impl QuoteFilter { /// Creates a new [QuoteFilter] from a "glob"-like pattern provided as input. /// No filtering is applied based on matching files' contents. pub fn new_from_glob(glob: String) -> Self { Self { path: glob, // The following regex matches *any* string: content: ".*".to_string(), _dates: vec![], } } // TODO: actually leverage async I/O fn read_files(&self) -> Result, ()> { use glob::glob; use grep::{regex, searcher::sinks}; let matcher = regex::RegexMatcher::new(&self.content).map_err(|_| ())?; let mut searcher = grep::searcher::Searcher::new(); let mut results = Vec::new(); for file in glob(&self.path).map_err(|_| ())? { let file = match file { Ok(file) => file, Err(_) => continue, }; let mut matched = false; let sink = sinks::Lossy(|_lnum, _line| { matched = true; Ok(false) }); let search_result = searcher.search_path(&matcher, &file, sink); if !matched || search_result.is_err() { continue; } let contents = std::fs::read_to_string(file).map_err(|_| ())?; results.push(contents.trim().into()); } Ok(results) } } } pub mod queue { use redis::AsyncCommands; use std::collections::VecDeque; use super::*; // Messages to interact with the quote queue #[derive(Debug)] pub struct DequeueQuote; pub type DequeueReply = Result, ()>; impl Message for DequeueQuote { type Reply = DequeueReply; } #[derive(Debug)] pub struct EnqueueQuotes(pub Vec); /// In case of a successful enqueue, no further data is required; /// however, if an error occurs partway through, we must return /// the numbers of quotes to have already been pushed up to that point. /// A [usize] is used as that is the maximum amount of items a [Vec] /// of quotes within [EnqueueQuotes] can contain. pub type EnqueueReply = Result<(), usize>; impl Message for EnqueueQuotes { type Reply = EnqueueReply; } /// Subtrait of Actor which specifically /// denotes actors that can handle all relevant queue messages. pub trait QueueManager: Handle + Handle {} impl QueueManager for T where T: Handle + Handle {} /// A basic implementation of an in-memory queue of quotes. /// Its contents are *not* persisted across application restarts, so it /// is only suited for testing purposes. #[derive(Default)] pub struct MemoryQueueStorage { quotes: VecDeque, } impl MemoryQueueStorage { pub fn new() -> Self { Self { quotes: VecDeque::new(), } } } #[async_trait::async_trait(?Send)] impl Handle for MemoryQueueStorage { async fn handle(&mut self, msg: EnqueueQuotes) -> Result<(), usize> { // Note: as `.push_back` into a `VecDeque` is an infallible operation, // we never need to worry about failure cases in this implementation. for q in msg.0 { self.quotes.push_back(q); } Ok(()) } } #[async_trait::async_trait(?Send)] impl Handle for MemoryQueueStorage { async fn handle(&mut self, _msg: DequeueQuote) -> Result, ()> { // Note: this can never fail, since the quotes are stored in memory Ok(self.quotes.pop_front()) } } /// Implementation of persistent quote queue storage through a Redis server. pub struct RedisQueueStorage { /// Key that points to the Redis queue to be used for fetching and storing quotes. queue_key: String, /// Underlying client from which to instantiate Redis connections. client: redis::Client, } impl RedisQueueStorage { /// The default queue key we use to fetch and store Redis quote. const DEFAULT_KEY: &str = "queue:default"; pub fn new(client: redis::Client, queue_key: String) -> Self { Self { queue_key, client } } pub fn new_from_env() -> Result { let client = redis::Client::open( std::env::var("REDIS_URL").unwrap_or("redis://localhost".to_string()), ) .map_err(|_| ())?; let queue_key = std::env::var("REDIS_KEY").unwrap_or(Self::DEFAULT_KEY.to_string()); Ok(Self { queue_key, client }) } async fn con(&self) -> Result { self.client .get_multiplexed_async_connection() .await .map_err(|e| { tracing::error!(error = e.to_string(), "Could not connect to Redis server"); () }) } } #[async_trait::async_trait(?Send)] impl Handle for RedisQueueStorage { #[tracing::instrument(name = "redis::enqueue_many", skip(self, msg))] async fn handle(&mut self, msg: EnqueueQuotes) -> Result<(), usize> { tracing::debug!( to_enqueue = msg.0.len(), "Enqueuing {} quotes...", msg.0.len() ); // We retrieve the connection object ahead of time to error-out early // in case of connection issues and avoid the overhead of fetching // a connection every iteration of the loop. let mut con = self.con().await.map_err(|_| 0usize)?; for (idx, q) in msg.0.iter().enumerate() { let _: () = con .rpush(&self.queue_key, q.get()) .await // If something goes wrong, we return the number of quotes // **already successfully queued** as part of our error message. .map_err(|e| { tracing::error!( error = e.to_string(), queued_so_far = idx, total_to_queue = msg.0.len(), "Could not enqueue all quotes to redis" ); idx })?; } Ok(()) } } #[async_trait::async_trait(?Send)] impl Handle for RedisQueueStorage { #[tracing::instrument(name = "redis::dequeue_one", skip(self, _msg))] async fn handle(&mut self, _msg: DequeueQuote) -> Result, ()> { tracing::debug!("Dequeuing quote..."); let quote: Result, ()> = self .con() .await? .lpop(&self.queue_key, None) .await .map_err(|e| { tracing::error!(error = e.to_string(), "Could not dequeue quote from Redis"); () }); match quote { Ok(Some(quote)) => Ok(Some(Quote::from(quote))), // We need to separate these next two cases, // as the type of quote: `Result>` // does not match the return type: `Result>` Ok(None) => Ok(None), Err(()) => Err(()), } } } } pub struct QuoteCycle { rng: rng::PrngState, source_manager: S, queue_manager: Q, } impl QuoteCycle { pub fn new(rng: rng::PrngState, source_manager: S, queue_manager: Q) -> Self { Self { rng, source_manager, queue_manager, } } pub fn with_thread_rng(source_manager: S, queue_manager: Q) -> Self { Self { rng: rng::PrngState::from_thread_rng(), source_manager, queue_manager, } } } /// A message to [QuoteCycle] to fetch one more quote from its storage. #[derive(Debug)] pub struct FetchQuote; impl Message for FetchQuote { type Reply = Result; } #[async_trait::async_trait(?Send)] impl Handle for QuoteCycle where S: source::SourceManager, Q: queue::QueueManager, { #[tracing::instrument(name = "quote_cycle::fetch", skip(self))] async fn handle(&mut self, _msg: FetchQuote) -> Result { // 1. We query our queue storage for the next quote tracing::debug!("Fetching new quotes..."); if let Some(next_quote) = self .queue_manager .handle(DequeueQuote) .await .map_err(|_| ())? { // if there is a quote, we simply return it and move on tracing::debug!( quote = next_quote.get().to_string(), "Quote dequeued immediately." ); return Ok(next_quote); } // 2. Otherwise, we must repopulate the queue through our source let mut refreshed_quotes = self.source_manager .handle(SourceQuotes) .await .map_err(|_| { tracing::error!("Could not retrieve new quotes from source"); () })?; tracing::debug!("Retrieved {} quotes.", refreshed_quotes.len()); // 3. We shuffle the newly-sourced quotes tracing::debug!("Shuffling quotes..."); self.rng.shuffle_slice(&mut refreshed_quotes); let refreshed_quotes = refreshed_quotes; // No longer mutable // TODO: Perhaps we should assert that the new quotes are non-empty? // 4. We enqueue the newly-sourced quotes... tracing::debug!("Enqueuing quotes..."); self.queue_manager .handle(EnqueueQuotes(refreshed_quotes)) .await .map_err(|_| { tracing::error!("Could not enqueue quotes to storage"); () })?; // 5. and, finally, we return the first among them. match self.queue_manager.handle(DequeueQuote).await { Ok(Some(q)) => Ok(q), Ok(None) => panic!("Newly-enqueued quotes should never be empty"), Err(_) => Err(()), } } } mod test { #[tokio::test] async fn memory_queue() { use super::Quote; use super::queue::*; use super::*; let mut queue_manager = MemoryQueueStorage::new(); let sample_quotes = ["Test no.1", "Test no.2", "Test no.3"]; queue_manager .handle(EnqueueQuotes( sample_quotes.iter().cloned().map(Quote::from).collect(), )) .await .expect("In-memory quote queue storage should be valid for insertion"); for text in sample_quotes.iter() { assert_eq!( *text, queue_manager .handle(DequeueQuote) .await .expect("In-memory queue storage should never panic on dequeue") .expect("In-memory queue storage should never be initialized as empty") .get() ); } } #[tokio::test] async fn memory_source() { use super::source::*; use super::*; let sample_quotes = ["Minie", "Miney", "Moe", "and", "some", "more"]; let mut source_manager = MemorySourceManager::new(sample_quotes); let quotes = source_manager .handle(SourceQuotes) .await .expect("In-memory quote queue storage should be valid for insertion"); assert_eq!( sample_quotes.as_slice(), quotes .into_iter() // Since [Quote] doesn't implement any Equality trait, // we map the strings into quotes instead .map(String::from) .collect::>() .as_slice(), ); } #[tokio::test] async fn memory_cycle() { use std::{collections::HashMap, ops::AddAssign}; use super::FetchQuote; use super::QuoteCycle; use super::queue::*; use super::source::*; use super::*; let sample_quotes = ["Minie", "Miney", "Moe"]; let mut cycle = { let source = MemorySourceManager::new(sample_quotes); let queue = MemoryQueueStorage::new(); QuoteCycle::with_thread_rng(source, queue) }; // We loop over `sample_quotes` twice to simulate the queue being exhausted fully, then re-sourced // Since the `cycle` manager will shuffle the quote sequence, we will verify that each // quote appears *exactly* `LOOPS` times throughout these iterations. const LOOPS: usize = 3; let mut quote_counts = HashMap::new(); for _ in 0..(sample_quotes.len() * LOOPS) { let next_quote = cycle.handle(FetchQuote).await.unwrap(); quote_counts .entry(next_quote.get().to_owned()) .or_insert(0) .add_assign(1); } let quote_counts = quote_counts; // no longer mut assert!( // Note: technically speaking, different quotes could contain equivalent strings, // which would make this test fail; a "more proper" invariant check would ensure // verify that all counts are a multiple of the amount of times `sample_quotes` was chained, // and that the sum of all counts equals the total number of times a `FetchQuote` message was sent. quote_counts.into_values().into_iter().all(|c| c == LOOPS) ); } }