Latest changes

Cargo.toml

···
 ]
 
 resolver = "2"
+
+[workspace.dependencies]
+miniserde = "0.1"

cfg-classify/src/lib.rs

···
 pub mod useful;
 
 pub trait CfgClassifyExt {
-#[cfg(feature = "ll")]
-    fn ll_parse_table(&self) -> ll::LlParseTable;
-#[cfg(feature = "lr")]
-    fn lr0_fsm_builder(&mut self) -> lr::Lr0FsmBuilder;
-#[cfg(feature = "lr")]
-    fn lr0_closure_builder(&mut self) -> lr::Lr0ClosureBuilder;
-    fn recursion(&self) -> recursive::Recursion;
+    #[cfg(feature = "ll")]
+    fn ll_parse_table(&self) -> ll::LlParseTable<'_>;
+    #[cfg(feature = "lr")]
+    fn lr0_fsm_builder(&mut self) -> lr::Lr0FsmBuilder<'_>;
+    #[cfg(feature = "lr")]
+    fn lr0_closure_builder(&mut self) -> lr::Lr0ClosureBuilder<'_>;
+    fn recursion(&self) -> recursive::Recursion<'_>;
     fn make_proper(&mut self) -> bool;
     fn usefulness(&mut self) -> useful::Usefulness;
     fn usefulness_with_roots(&mut self, roots: &[Symbol]) -> useful::Usefulness;
···
 
 impl CfgClassifyExt for Cfg {
     #[cfg(feature = "ll")]
-    fn ll_parse_table(&self) -> ll::LlParseTable {
+    fn ll_parse_table(&self) -> ll::LlParseTable<'_> {
         ll::LlParseTable::new(self)
     }
 
-    fn recursion(&self) -> recursive::Recursion {
+    fn recursion(&self) -> recursive::Recursion<'_> {
         recursive::Recursion::new(self)
     }
 
     #[cfg(feature = "lr")]
-    fn lr0_fsm_builder(&mut self) -> lr::Lr0FsmBuilder {
+    fn lr0_fsm_builder(&mut self) -> lr::Lr0FsmBuilder<'_> {
         lr::Lr0FsmBuilder::new(self)
     }
 
     #[cfg(feature = "lr")]
-    fn lr0_closure_builder(&mut self) -> lr::Lr0ClosureBuilder {
+    fn lr0_closure_builder(&mut self) -> lr::Lr0ClosureBuilder<'_> {
         lr::Lr0ClosureBuilder::new(self)
     }
 

cfg-grammar/Cargo.toml

···
 name = "cfg_grammar"
 
 [dependencies]
-bit-vec = "0.8"
+bit-vec = { version = "0.8", features = ["miniserde"] }
 cfg-symbol = { version = "0.0.1", path = "../cfg-symbol/" }
 cfg-history = { version = "0.0.1", path = "../cfg-history/" }
 smallvec = { version = "1.13", features = ["const_generics"], optional = true }
···
 
 # serialize
 serde = { version = "1.0", features = ["derive"], optional = true }
-miniserde = { version = "0.1", optional = true }
+miniserde = { workspace = true }
 
 [features]
 serde = ["dep:serde", "cfg-symbol/serde", "smallvec/serde", "bit-vec/serde", "cfg-history/serde"]
-miniserde = ["dep:miniserde", "cfg-symbol/miniserde", "bit-vec/miniserde"]

cfg-grammar/src/cfg.rs

···
     }
 
     /// Starts building a new rule.
-    pub fn rule(&mut self, lhs: Symbol) -> RuleBuilder {
+    pub fn rule(&mut self, lhs: Symbol) -> RuleBuilder<'_> {
         RuleBuilder::new(self).rule(lhs)
     }
 
     /// Starts building a new precedenced rule.
-    pub fn precedenced_rule(&mut self, lhs: Symbol) -> PrecedencedRuleBuilder {
+    pub fn precedenced_rule(&mut self, lhs: Symbol) -> PrecedencedRuleBuilder<'_> {
         PrecedencedRuleBuilder::new(self, lhs)
     }
 

cfg-grammar/src/symbol_bit_set.rs

···
 use crate::local_prelude::*;
 
 /// A set of symbols in the form of a bit vector.
-#[cfg_attr(feature = "miniserde", derive(miniserde::Serialize, miniserde::Deserialize))]
-#[derive(Clone, Debug)]
+#[derive(miniserde::Serialize, miniserde::Deserialize, Clone, Debug)]
 pub struct SymbolBitSet {
     bit_vec: BitVec,
 }

cfg-history/Cargo.toml

···
 cfg-symbol = { version = "0.0.1", path = "../cfg-symbol/" }
 smallvec = { version = "1.15", optional = true }
 serde = { version = "1.0", features = ["derive"], optional = true }
-miniserde = { version = "0.1", optional = true }
+miniserde = { workspace = true }
+nanoserde = { version = "0.2", optional = true }
 
 [features]
 default = ["earley"]

cfg-history/src/earley/mod.rs

···
 
 use rule_dot::RuleDot;
 
-#[cfg_attr(feature = "miniserde", derive(miniserde::Serialize, miniserde::Deserialize))]
-#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(feature = "nanoserde", derive(nanoserde::SerBin, nanoserde::DeBin))]
+#[derive(miniserde::Serialize, miniserde::Deserialize, Copy, Clone, Default, Debug, Eq, PartialEq)]
 pub struct ExternalOrigin { pub id: u32 }
-#[cfg_attr(feature = "miniserde", derive(miniserde::Serialize, miniserde::Deserialize))]
-#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(feature = "nanoserde", derive(nanoserde::SerBin, nanoserde::DeBin))]
+#[derive(miniserde::Serialize, miniserde::Deserialize, Copy, Clone, Default, Debug, Eq, PartialEq)]
 pub struct EventId { pub id: u32 }
-#[cfg_attr(feature = "miniserde", derive(miniserde::Serialize, miniserde::Deserialize))]
-#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(feature = "nanoserde", derive(nanoserde::SerBin, nanoserde::DeBin))]
+#[derive(miniserde::Serialize, miniserde::Deserialize, Copy, Clone, Default, Debug, Eq, PartialEq)]
 pub struct MinimalDistance { pub distance: u32 }
 pub type NullingEliminated = Option<(Symbol, bool)>;
-#[cfg_attr(feature = "miniserde", derive(miniserde::Serialize, miniserde::Deserialize))]
-#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(feature = "nanoserde", derive(nanoserde::SerBin, nanoserde::DeBin))]
+#[derive(miniserde::Serialize, miniserde::Deserialize, Copy, Clone, Default, Debug, Eq, PartialEq)]
 pub struct ExternalDottedRule {
     id: u32,
     pos: u32,

cfg-load/src/advanced.rs

···
 #![deny(unsafe_code)]
 
 use cfg_history::RootHistoryNode;
-use cfg_regexp::{CfgRegexpExt, LexerMap};
+use cfg_regexp::CfgRegexpExt;
 use cfg_symbol_bit_matrix::Remap;
 use tiny_earley::{grammar, forest, Recognizer, Symbol};
 
 use cfg_grammar::{Cfg, SymbolBitSet};
 use cfg_sequence::CfgSequenceExt;
-use std::{collections::{BTreeSet, HashMap, HashSet}, convert::AsRef, fmt, iter, str::Chars};
+use std::{collections::{BTreeMap, BTreeSet, HashMap, HashSet}, convert::AsRef, iter, str::Chars};
 
 use elsa::FrozenIndexSet;
+
+pub use cfg_regexp::LexerMap;
 
 use crate::LoadError;
 pub struct StringInterner {
···
 struct LexerVal(Vec<Rule>);
 
 struct Evaluator {
-    symbols: [Symbol; 23],
+    symbols: [Symbol; 24],
     tokens: Vec<(Token, usize, usize)>,
 }
 
···
 
     fn leaf(&self, terminal: Symbol, values: u32) -> Self::Elem {
         #[allow(unused_variables)]
-        let [start, rule, alt, rhs, bnf_op, ident, pipe, op_mul, op_plus, semicolon, fragment, string, decl, action, lexer_keyword, lexer, lbrace, rbrace, rules, gt_op, lparen, rparen, alt2] =
+        let [start, rule, alt, rhs, bnf_op, ident, pipe, op_mul, op_plus, semicolon, fragment, string, decl, action, lexer_keyword, lexer, lbrace, rbrace, rules, gt_op, lparen, rparen, alt2, tilde_op] =
             self.symbols;
         if terminal == ident {
             self.tokens[values as usize].0.ident()
···
 
     fn product(&self, action_num: u32, args: Vec<Self::Elem>) -> Self::Elem {
         #[allow(unused_variables)]
-        let [start, rule, alt, rhs, bnf_op, ident, pipe, op_mul, op_plus, semicolon, fragment, string, decl, action, lexer_keyword, lexer, lbrace, rbrace, rules, gt_op, lparen, rparen, alt2] =
+        let [start, rule, alt, rhs, bnf_op, ident, pipe, op_mul, op_plus, semicolon, fragment, string, decl, action, lexer_keyword, lexer, lbrace, rbrace, rules, gt_op, lparen, rparen, alt2, tilde_op] =
             self.symbols;
         // let mut iter = args.into_iter();
         match (
···
             (12, Value::Alt(alt), Value::Ident(action), _) => {
                 Value::Alt2(alt, Some(action))
             }
-            // alt ::= alt fragment;
-            (13, Value::Alt(mut alt), Value::Fragment(fragment), _) => {
+            // alt ::= alt tilde_op fragment;
+            (13, Value::Alt(mut alt), _, Value::Fragment(fragment)) => {
                 alt.push(fragment);
                 Value::Alt(alt)
             }
···
     RParen,
     GtOp,
     LexerKeyword,
+    TildeOp,
     Error(usize, usize),
 }
 
···
                 self.advance();
                 Token::Mul
             }
+            '~' => {
+                self.advance();
+                Token::TildeOp
+            }
             '(' => {
                 self.advance();
                 Token::LParen
···
 }
 
 pub trait CfgLoadAdvancedExt {
-    fn load_advanced(grammar: &str) -> Result<(Cfg, LexerMap, SymbolBitSet), LoadError>;
+    fn load_advanced(grammar: &str) -> Result<AdvancedGrammar, LoadError>;
 }
 
 // fn lexeme_set(cfg: &Cfg, lexeme_origin: usize) -> Result<SymbolBitSet, LoadError> {
···
 //     }
 // }
 
+pub struct AdvancedGrammar {
+    pub cfg: Cfg,
+    pub lexer_map: LexerMap,
+    pub sbs: SymbolBitSet,
+    pub actions: BTreeMap<usize, Option<String>>,
+}
+
 impl CfgLoadAdvancedExt for Cfg {
-    fn load_advanced(grammar: &str) -> Result<(Cfg, LexerMap, SymbolBitSet), LoadError> {
+    fn load_advanced(grammar: &str) -> Result<AdvancedGrammar, LoadError> {
         use tiny_earley::Grammar;
         let bnf_grammar = grammar! {
-            S = [start, rule, alt, rhs, bnf_op, ident, pipe, op_mul, op_plus, semicolon, fragment, string, decl, action, lexer_keyword, lexer, lbrace, rbrace, rules, gt_op, lparen, rparen, alt2]
+            S = [start, rule, alt, rhs, bnf_op, ident, pipe, op_mul, op_plus, semicolon, fragment, string, decl, action, lexer_keyword, lexer, lbrace, rbrace, rules, gt_op, lparen, rparen, alt2, tilde_op]
             R = {
                 start ::= start decl; // 2
                 start ::= decl; // 3
···
                 rhs ::= alt2; // 10
                 alt2 ::= alt; // 11
                 alt2 ::= alt action; // 12
-                alt ::= alt fragment; // 13
+            alt ::= alt tilde_op fragment; // 13
                 alt ::= fragment; // 14
                 fragment ::= ident op_plus; // 15
                 fragment ::= ident op_mul; // 16
···
         };
         let symbols = bnf_grammar.symbols();
         #[allow(unused_variables)]
-        let [start, rule, alt, rhs, bnf_op, ident, pipe, op_mul, op_plus, semicolon, fragment, string, decl, action, lexer_keyword, lexer, lbrace, rbrace, rules, gt_op, lparen, rparen, alt2] = bnf_grammar.symbols();
+        let [start, rule, alt, rhs, bnf_op, ident, pipe, op_mul, op_plus, semicolon, fragment, string, decl, action, lexer_keyword, lexer, lbrace, rbrace, rules, gt_op, lparen, rparen, alt2, tilde_op] = bnf_grammar.symbols();
         let mut recognizer = Recognizer::new(&bnf_grammar);
         let tokens = Lexer::tokenize(grammar);
         for (i, &(ref ch, line, col)) in tokens.iter().enumerate() {
···
                 Token::LParen => lparen,
                 Token::RParen => rparen,
                 Token::GtOp => gt_op,
+                Token::TildeOp => tilde_op,
                 Token::LexerKeyword => lexer_keyword,
                 Token::Whitespace => continue,
                 &Token::Error(line_no, col_no) => return Err(LoadError::Parse { reason: "failed to tokenize".to_string(), line: line_no as u32, col: col_no as u32, token: None }),
···
             let mut intern_empty = true;
             let mut lexer_classes = LexerMap::new();
             let mut lhs_in_parser = HashSet::new();
+            let mut actions = BTreeMap::new();
             // check_for_lexical_error(&rules, lexer.as_ref());
             for (idx, (rule, is_lexer)) in rules.into_iter().zip(iter::repeat(false)).chain(lexer.unwrap_or(LexerVal(vec![])).0.into_iter().zip(iter::repeat(true))).enumerate() {
                 let lhs = intern.get_or_intern(&rule.lhs[..]);
···
                             }
                             match &*arg {
                                 &Fragment::Call { ref func, ref arg } => {
-                                    return Err(LoadError::Lex { reason: format!("expected Regex(string)") });
+                                    return Err(LoadError::Lex { reason: format!("expected Regex(string), found {:?}({:?})", func, arg) });
                                 }
                                 &Fragment::Lex { ref string } => {
                                     let (mut regexp_cfg, classes) = Cfg::from_regexp(string).map_err(|err| LoadError::Lex { reason: err.to_string() })?;
                                     let mut remap = Remap::new(&mut regexp_cfg);
                                     let mut sym_map = HashMap::new();
-                                    for (class, sym) in classes.0 {
-                                        let new_sym = *lexer_classes.0.entry(class).or_insert_with(|| cfg.next_sym(None));
+                                    for (class, sym) in classes.classes {
+                                        let new_sym = *lexer_classes.classes.entry(class).or_insert_with(|| cfg.next_sym(None));
                                         sym_map.insert(sym, new_sym);
                                     }
                                     remap.remap_symbols(|sym| {
···
                                     Ok(regexp_cfg.roots()[0])
                                 }
                                 &Fragment::Rhs { ref ident, rep } => {
-                                    return Err(LoadError::Lex { reason: format!("expected Regex(string)") });
+                                    return Err(LoadError::Lex { reason: format!("expected Regex(string), found {:?} repeated {:?}", ident, rep) });
                                 }
                             }
                         }
···
                             let rhs_sym = *sym_map.entry(id).or_insert_with(|| {
                                 cfg.sym_source_mut().next_sym(Some(name.clone().into()))
                             });
-                            let child: Vec<_> = string.chars().map(|ch| *lexer_classes.0.entry(ch.into()).or_insert_with(|| cfg.next_sym(None))).collect();
+                            let child: Vec<_> = string.chars().map(|ch| *lexer_classes.classes.entry(ch.into()).or_insert_with(|| cfg.next_sym(None))).collect();
                             cfg.rule(rhs_sym).rhs(&child[..]);
                             set_of_lexical.insert(rhs_sym);
                             Ok(rhs_sym)
···
                     set_of_lexical.insert(lhs_sym);
                 }
                 cfg.rule(lhs_sym).history(RootHistoryNode::Origin { origin: idx + 1 }.into()).rhs(rhs_syms?);
+                actions.insert(idx + 1, rule.action);
             }
             let mut sbs = SymbolBitSet::new();
             sbs.reset(cfg.sym_source());
             for sym in set_of_lexical {
                 sbs.set(sym, true);
             }
-            Ok((cfg, lexer_classes, sbs))
+            Ok(AdvancedGrammar {
+                cfg,
+                lexer_map: lexer_classes,
+                sbs,
+                actions,
+            })
         } else {
             return Err(LoadError::Eval { reason: format!("evaluation failed: Expected Value::Rules, got {:?}", result) });
         }

cfg-load/src/basic.rs

···
 
 use cfg_grammar::Cfg;
 use cfg_sequence::CfgSequenceExt;
-use std::{collections::HashMap, convert::AsRef, fmt::{self, Write}, str::Chars};
+use std::{collections::HashMap, convert::AsRef, fmt::Write, str::Chars};
 
 use elsa::FrozenIndexSet;
 

cfg-load/src/lib.rs

···
 
 use std::fmt;
 
-pub use crate::basic::{CfgLoadExt};
-pub use crate::advanced::{CfgLoadAdvancedExt};
+pub use crate::basic::CfgLoadExt;
+pub use crate::advanced::CfgLoadAdvancedExt;
 
 #[derive(Debug, Clone)]
 pub enum LoadError {

cfg-load/tests/test_load.rs

···
 
 #[test]
 fn test_load_advanced() {
-    let (cfg, _, bs) = Cfg::load_advanced(r#"
-        start ::= a b c d;
+    let grammar = Cfg::load_advanced(r#"
+        start ::= a ~ b ~ c ~ d;
         a ::= y;
     "#).unwrap();
-    assert_eq!(cfg.rules().count(), 2);
-    let (cfg, _, bs) = Cfg::load_advanced(r#"
-        start ::= a b c d;
+    assert_eq!(grammar.cfg.rules().count(), 2);
+    let grammar = Cfg::load_advanced(r#"
+        start ::= a ~ b ~ c ~ d;
         a ::= "y";
     "#).unwrap();
-    assert_eq!(cfg.rules().count(), 3);
-    let (cfg, _, bs) = Cfg::load_advanced(r#"
-        start ::= a b c d;
+    assert_eq!(grammar.cfg.rules().count(), 3);
+    let grammar = Cfg::load_advanced(r#"
+        start ::= a ~ b ~ c ~ d;
         a ::= "y";
         lexer {
             x ::= "z";
         }
     "#).unwrap();
-    assert_eq!(cfg.rules().count(), 5);
-    let (cfg, lm, bs) = Cfg::load_advanced(r#"
-        start ::= a b c d;
+    assert_eq!(grammar.cfg.rules().count(), 5);
+    let grammar = Cfg::load_advanced(r#"
+        start ::= a ~ b ~ c ~ d;
         a ::= "y";
         lexer {
             x ::= "zzt";
             x ::= Regexp("test");
         }
     "#).unwrap();
-    assert_eq!(cfg.rules().count(), 7);
-    assert_eq!(format!("{:?}", lm.0.keys().flat_map(|cl| cl.iter().next()).collect::<Vec<_>>()), "[('e', 'f'), ('s', 't'), ('t', 'u'), ('y', 'z'), ('z', '{')]");
-    assert_eq!(cfg.to_bnf(), r#"start ::= a b c d;
+    assert_eq!(grammar.cfg.rules().count(), 7);
+    assert_eq!(format!("{:?}", grammar.lexer_map.classes.keys().flat_map(|cl| cl.iter().next()).collect::<Vec<_>>()), "[('e', 'f'), ('s', 't'), ('t', 'u'), ('y', 'z'), ('z', '{')]");
+    assert_eq!(grammar.cfg.to_bnf(), r#"start ::= a b c d;
 __lex0 ::= g6;
 a ::= __lex0;
 __lex1 ::= g9 g9 g10;
···
 g13 ::= g10 g11 g12 g10;
 x ::= g13;
 "#);
-    assert_eq!(bs.iter().count(), 3);
+    assert_eq!(grammar.sbs.iter().count(), 3);
 }
 
 #[test]
 fn test_err() {
     assert!(Cfg::load_advanced(r#"
-        start ::= a b c d;
+        start ::= a ~ b ~ c ~ d;
         x ::= "y";
         lexer {
             x ::= "zzt";
···
 
 #[test]
 fn test_big() {
-    let (cfg, lm, bs) = Cfg::load_advanced(r#"
-        start ::= a b c d;
+    let grammar = Cfg::load_advanced(r#"
+        start ::= a ~ b ~ c ~ d;
         a ::= "y";
         lexer {
             x ::= "zzt";
             x ::= Regexp("test(er)?");
         }
     "#).unwrap();
-    assert_eq!(cfg.rules().count(), 11);
-    assert_eq!(format!("{:?}", lm.0.keys().flat_map(|cl| cl.iter().next()).collect::<Vec<_>>()), "[('e', 'f'), ('r', 's'), ('s', 't'), ('t', 'u'), ('y', 'z'), ('z', '{')]");
-    assert_eq!(cfg.to_bnf(), r#"start ::= a b c d;
+    assert_eq!(grammar.cfg.rules().count(), 11);
+    assert_eq!(format!("{:?}", grammar.lexer_map.classes.keys().flat_map(|cl| cl.iter().next()).collect::<Vec<_>>()), "[('e', 'f'), ('r', 's'), ('s', 't'), ('t', 'u'), ('y', 'z'), ('z', '{')]");
+    assert_eq!(grammar.cfg.to_bnf(), r#"start ::= a b c d;
 __lex0 ::= g6;
 a ::= __lex0;
 __lex1 ::= g9 g9 g10;
···
 g17 ::= g10 g11 g13 g10 g15;
 x ::= g17;
 "#);
-    assert_eq!(bs.iter().count(), 3);
+    assert_eq!(grammar.sbs.iter().count(), 3);
 }

cfg-predict-distance/src/lib.rs

···
 //! Calculation of minimum distance from one part of the grammar to another.
 
 use cfg_grammar::*;
-use cfg_history::{earley::History, HistoryId};
 use cfg_symbol::Symbol;
 
 /// Calculation of minimum distance from one part of the grammar to another.

cfg-regexp/Cargo.toml

···
 
 [dependencies]
 regex-syntax = "0.8"
+regex-automata = "0.4"
 log = "0.4"
 cfg-grammar = { version = "0.0.1", path = "../cfg-grammar/" }
 cfg-sequence = { version = "0.0.1", path = "../cfg-sequence/" }

cfg-regexp/src/lib.rs

···
 use cfg_grammar::Cfg;
 use cfg_sequence::CfgSequenceExt;
 use cfg_symbol::Symbol;
-use regex_syntax::hir::{self, Class, Hir, HirKind};
+use regex_syntax::hir::{Class, Hir, HirKind};
 use regex_syntax::Parser;
 
 pub trait CfgRegexpExt: Sized {
···
 }
 
 #[derive(Debug, Clone)]
-pub struct LexerMap(pub BTreeMap<LexerClasses, Symbol>);
+pub struct LexerMap {
+    pub classes: BTreeMap<LexerClasses, Symbol>,
+    ascii: Vec<Vec<Symbol>>,
+    ranges: BTreeMap<char, Vec<Symbol>>,
+}
 
 impl LexerMap {
     pub fn new() -> Self {
-        LexerMap(BTreeMap::new())
+        LexerMap {
+            classes: BTreeMap::new(),
+            ascii: vec![],
+            ranges: BTreeMap::new(),
+        }
+    }
+
+    pub fn compute(&mut self) {
+        let mut result = vec![vec![]; 256];
+        for (lexer_classes, &symbol) in &self.classes {
+            for &class in &lexer_classes.set {
+                if class.0.is_ascii() {
+                    for ascii in class.0 as u32 ..= (class.1 as u32).min(256) {
+                        result[ascii as usize].push(symbol);
+                    }
+                }
+            }
+        }
+        self.ascii = result;
+        let mut ranges = BTreeMap::new();
+        for (lexer_classes, &symbol) in &self.classes {
+            for &class in &lexer_classes.set {
+                ranges.entry(class.0).or_insert(vec![]).push((true, symbol));
+                ranges.entry(char::from_u32(class.1 as u32 + 1).unwrap()).or_insert(vec![]).push((false, symbol));
+            }
+        }
+        let mut result = BTreeMap::new();
+        let mut work = BTreeSet::new();
+        for (ch, changes) in ranges {
+            for (is_added, symbol) in changes {
+                if is_added {
+                    work.insert(symbol);
+                } else {
+                    work.remove(&symbol);
+                }
+            }
+            result.entry(ch).or_insert(vec![]).extend(work.iter().copied());
+        }
+        self.ranges = result;
+    }
+
+    pub fn get(&self, ch: char) -> &[Symbol] {
+        if ch.is_ascii() {
+            &self.ascii[ch as usize][..]
+        } else {
+            self.ranges.range(..=ch).next_back().map(|(_, v)| &v[..]).unwrap_or(&[])
+        }
     }
 }
 
 impl Translator {
     fn cfg_from_hir(hir: Hir) -> (Cfg, LexerMap) {
         let cfg = Cfg::new();
-        let class_map = LexerMap(BTreeMap::new());
+        let class_map = LexerMap::new();
         let mut this = Self { cfg, class_map };
         let x = this.walk_hir(&hir, 0);
         let lhs = match (x.len(), x.get(0).map_or(0, |y| y.len())) {
···
             HirKind::Literal(lit) => {
                 let mut syms = vec![];
                 for &byte in &lit.0 {
-                    syms.push(*self.class_map.0.entry(byte.into()).or_insert_with(|| self.cfg.next_sym(None)));
+                    syms.push(*self.class_map.classes.entry(byte.into()).or_insert_with(|| self.cfg.next_sym(None)));
                 }
                 println!("{indent}Literal: {:?}", lit);
                 vec![syms]
             }
             HirKind::Class(class) => {
-                let sym = *self.class_map.0.entry(class.clone().into()).or_insert_with(|| self.cfg.next_sym(None));
+                let sym = *self.class_map.classes.entry(class.clone().into()).or_insert_with(|| self.cfg.next_sym(None));
                 println!("{indent}Class: {:?}", class);
                 vec![vec![sym]]
             }
···
                 println!("{indent}Concat:");
                 let mut result = vec![];
                 for expr in exprs {
-                    let mut x = self.walk_hir(expr, depth + 1);
+                    let x = self.walk_hir(expr, depth + 1);
                     match x.len() {
                         0 => {}
                         1 => {
···
                 }
                 alternatives
             }
-            HirKind::Look(look) => {
+            HirKind::Look(_look) => {
                 unimplemented!()
             }
             HirKind::Empty => {
···
 mod tests {
     use super::*;
 
-    use regex_syntax::hir::{self, Hir, HirKind};
-    use regex_syntax::Parser;
-
     #[test]
     fn it_works() {
         let result = add(2, 2);
···
 
 
         let pattern = r"(?i)(foo|bar)\d+";
-        let (result, map) = Cfg::from_regexp(pattern).unwrap();
-        assert_eq!(result.rules().count(), 4);
-        assert_eq!(map.0, BTreeMap::new());
+        let (result, mut map) = Cfg::from_regexp(pattern).unwrap();
+        assert_eq!(result.rules().count(), 5);
+        map.compute();
+        assert_eq!(map.get('b').len(), 1);
+        assert_eq!(map.get('c').len(), 0);
+        assert_eq!(map.get('B').len(), 1);
+        assert_eq!(map.get('D').len(), 0);
+        assert_eq!(map.get('o').len(), 1);
+        assert_eq!(map.get('🯰').len(), 1);
+        assert_eq!(map.get('🯹').len(), 1);
     }
 }

cfg-sequence/Cargo.toml

···
 
 [features]
 serde = ["cfg-symbol/serde"]
-miniserde = ["cfg-symbol/miniserde"]

cfg-sequence/src/builder.rs

···
 use crate::destination::SequenceDestination;
 use crate::{Separator, Sequence};
 use cfg_history::earley::History;
-use cfg_history::HistoryId;
 use cfg_symbol::Symbol;
 
 /// Sequence rule builder.

cfg-sequence/src/ext.rs

···
 use crate::{builder::SequenceRuleBuilder, destination::SequenceDestination, rewrite::SequencesToProductions};
 
 pub trait CfgSequenceExt {
-    fn sequence(&mut self, lhs: Symbol) -> SequenceRuleBuilder<SequencesToProductions>;
+    fn sequence(&mut self, lhs: Symbol) -> SequenceRuleBuilder<SequencesToProductions<'_>>;
 }
 
 impl CfgSequenceExt for Cfg {
-    fn sequence(&mut self, lhs: Symbol) -> SequenceRuleBuilder<SequencesToProductions> {
+    fn sequence(&mut self, lhs: Symbol) -> SequenceRuleBuilder<SequencesToProductions<'_>> {
         SequencesToProductions::new(self).sequence(lhs)
     }
 }

cfg-sequence/src/lib.rs

···
 
 use std::ops::{Bound, RangeBounds};
 
-use cfg_history::{earley::History, HistoryId};
+use cfg_history::earley::History;
 use cfg_symbol::Symbol;
 
 use self::Separator::*;

cfg-symbol-bit-matrix/src/remap_symbols.rs

···
 }
 
 pub trait CfgRemapSymbolsExt {
-    fn remap(&mut self) -> Remap;
+    fn remap(&mut self) -> Remap<'_>;
 }
 
 impl CfgRemapSymbolsExt for Cfg {
-    fn remap(&mut self) -> Remap {
+    fn remap(&mut self) -> Remap<'_> {
         Remap::new(self)
     }
 }

cfg-symbol/Cargo.toml

···
 [dependencies]
 # serialize
 serde = { version = "1.0", features = ["derive"], optional = true }
-miniserde = { version = "0.1", optional = true }
+miniserde = { workspace = true }
+nanoserde = { version = "0.2", optional = true }

cfg-symbol/src/intern.rs

···
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 
-#[cfg(feature = "miniserde")]
-use miniserde::{Deserialize as MiniDeserialize, Serialize as MiniSerialize};
-
 /// Contains maps for translation between internal and external symbols.
 #[derive(Clone, Default, Debug)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "miniserde", derive(MiniSerialize, MiniDeserialize))]
+#[derive(miniserde::Serialize, miniserde::Deserialize)]
 pub struct Mapping {
     /// An array of internal symbols, indexed by external symbol ID.
     pub to_internal: Vec<Option<Symbol>>,

cfg-symbol/src/source.rs

···
 }
 
 /// A source of numeric symbols.
-#[cfg_attr(feature = "miniserde", derive(miniserde::Serialize, miniserde::Deserialize))]
+#[derive(miniserde::Serialize, miniserde::Deserialize)]
 #[derive(Clone, Debug, Default)]
 pub struct SymbolSource<T: SymbolPrimitive = NonZeroU32> {
     next_symbol: Symbol<T>,
···
         self.names.push(name.map(|cow| cow.into()));
         ret
     }
-    pub fn name_of(&self, sym: Symbol) -> Cow<str> {
+    pub fn name_of(&self, sym: Symbol) -> Cow<'_, str> {
         match self.names.get(sym.usize()) {
             Some(Some(name)) => {
                 Cow::Borrowed(&name.name[..])
···
     }
 }
 
-#[cfg(feature = "miniserde")]
 mod miniserde_impls {
-    use crate::{Symbol, SymbolSource, SymbolPrimitive};
     use super::SymbolName;
     use std::rc::Rc;
     use miniserde::de::{Deserialize, Visitor};
     use miniserde::{de, ser, Serialize};
-    use miniserde::{make_place, Error, Result};
+    use miniserde::{make_place, Result};
 
     make_place!(Place);
 
···
     }
 
     impl Serialize for SymbolName {
-        fn begin(&self) -> ser::Fragment {
+        fn begin(&self) -> ser::Fragment<'_> {
             ser::Fragment::Str(self.name.to_string().into())
         }
     }

cfg-symbol/src/symbol.rs

···
     }
 }
 
-#[cfg(feature = "miniserde")]
 mod miniserde_impls {
     use super::{Symbol, SymbolPrimitive};
     use std::num::NonZeroU32;
···
     impl<T: SymbolPrimitive> Visitor for Place<Symbol<T>> {
         fn nonnegative(&mut self, n: u64) -> Result<()> {
             if n < T::MAX as u64 {
-                if let Some(Ok(nonzero_num)) = NonZeroU32::new(n as u32).map(|n| TryInto::<T>::try_into(n)) {
+                if let Some(Ok(nonzero_num)) = NonZeroU32::new((n + 1) as u32).map(|n| TryInto::<T>::try_into(n)) {
                     self.out = Some(Symbol { n: nonzero_num });
                     Ok(())
                 } else {
···
     }
 
     impl<T: SymbolPrimitive> Serialize for Symbol<T> {
-        fn begin(&self) -> ser::Fragment {
+        fn begin(&self) -> ser::Fragment<'_> {
             let n: u32 = (*self).into();
             ser::Fragment::U64(n as u64)
         }
     }
 }
+
+#[cfg(feature = "nanoserde")]
+impl nanoserde::DeBin for Symbol<NonZeroU32> {
+    fn de_bin(offset: &mut usize, bytes: &[u8]) -> Result<Self, nanoserde::DeBinErr> {
+        Ok(Symbol {
+            n: u32::de_bin(offset, bytes)?.try_into().unwrap()
+        })
+    }
+}
+
+#[cfg(feature = "nanoserde")]
+impl nanoserde::SerBin for Symbol<NonZeroU32> {
+    fn ser_bin(&self, output: &mut Vec<u8>) {
+        u32::ser_bin(&self.n.get(), output);
+    }
+}

cfg/Cargo.toml

···
 [features]
 default = ["cfg-classify", "cfg-generate", "cfg-predict-sets", "cfg-predict-distance", "cfg-generate", "cfg-sequence", "cfg-history", "cfg-symbol-bit-matrix"]
 serde = ["cfg-grammar/serde"]
-miniserde = ["cfg-grammar/miniserde", "cfg-history/miniserde"]
 ll = ["cfg-classify/ll"]
 lr = ["cfg-classify/lr"]
 weighted-generation = ["cfg-generate/weighted", "cfg-generate", "rand"]

tokei.sh

···
+#!/usr/bin/bash
+tokei cfg{-grammar,-regexp,-load,-history,-sequence,-symbol,-symbol-bit-matrix,-predict-sets,}/src
+
+if [ $? -eq 127 ]; then
+  echo "please run cargo install tokei (exit code 127)"
+fi
+