clone detector (rabin-karp)

crates/core/src/cloner/detector.rs

···
+use crate::Result;
+use crate::cloner::rolling_hash::{RollingHash, token_hash};
+use crate::tokenizer::{Language, Token, Tokenizer};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+/// A detected code clone
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Clone {
+    /// Unique ID for this clone group
+    pub id: usize,
+    /// Number of tokens in the cloned sequence
+    pub length: usize,
+    /// All locations where this clone appears
+    pub locations: Vec<CloneLocation>,
+    /// Hash value of the clone (for deduplication)
+    #[serde(skip)]
+    pub hash: u64,
+}
+
+/// Location of a code clone
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct CloneLocation {
+    /// File path
+    pub file: PathBuf,
+    /// Starting line number
+    pub start_line: usize,
+    /// Ending line number
+    pub end_line: usize,
+}
+
+pub struct CloneDetector {
+    /// Minimum number of tokens to consider as a clone
+    _min_tokens: usize,
+    /// Window size for rolling hash
+    window_size: usize,
+}
+
+impl Default for CloneDetector {
+    fn default() -> Self {
+        Self { _min_tokens: 30, window_size: 30 }
+    }
+}
+
+impl CloneDetector {
+    pub fn new(min_tokens: usize) -> Self {
+        Self { _min_tokens: min_tokens, window_size: min_tokens }
+    }
+
+    /// Detect clones in a single file
+    pub fn detect_in_file(&self, source: &str, language: Language, file_path: PathBuf) -> Result<Vec<Clone>> {
+        let tokens = Tokenizer::new(source, language).tokenize()?;
+        let significant_tokens: Vec<&Token> = tokens.iter().filter(|t| t.token_type.is_significant()).collect();
+
+        if significant_tokens.len() < self.window_size {
+            return Ok(Vec::new());
+        }
+
+        let mut hash_map: HashMap<u64, Vec<(usize, usize)>> = HashMap::new();
+        let mut rh = RollingHash::new(self.window_size);
+
+        let token_hashes: Vec<u64> = significant_tokens.iter().map(|t| token_hash(&t.text)).collect();
+
+        rh.init(&token_hashes[0..self.window_size]);
+        let start_line = significant_tokens[0].line;
+        let end_line = significant_tokens[self.window_size - 1].line;
+        hash_map.entry(rh.get()).or_default().push((start_line, end_line));
+
+        for i in self.window_size..token_hashes.len() {
+            let hash = rh.roll(token_hashes[i - self.window_size], token_hashes[i]);
+            let start_line = significant_tokens[i - self.window_size + 1].line;
+            let end_line = significant_tokens[i].line;
+            hash_map.entry(hash).or_default().push((start_line, end_line));
+        }
+
+        let mut clones = Vec::new();
+        let mut clone_id = 0;
+
+        for (hash, locations) in hash_map {
+            if locations.len() > 1 {
+                clone_id += 1;
+                clones.push(Clone {
+                    id: clone_id,
+                    length: self.window_size,
+                    locations: locations
+                        .into_iter()
+                        .map(|(start, end)| CloneLocation { file: file_path.clone(), start_line: start, end_line: end })
+                        .collect(),
+                    hash,
+                });
+            }
+        }
+
+        Ok(clones)
+    }
+
+    /// Detect clones across multiple files
+    pub fn detect_across_files(&self, files: &[(PathBuf, String, Language)]) -> Result<Vec<Clone>> {
+        let mut global_hash_map: HashMap<u64, Vec<CloneLocation>> = HashMap::new();
+
+        for (file_path, source, language) in files {
+            let tokens = Tokenizer::new(source, *language).tokenize()?;
+            let significant_tokens: Vec<&Token> = tokens.iter().filter(|t| t.token_type.is_significant()).collect();
+
+            if significant_tokens.len() < self.window_size {
+                continue;
+            }
+
+            let mut rh = RollingHash::new(self.window_size);
+
+            let token_hashes: Vec<u64> = significant_tokens.iter().map(|t| token_hash(&t.text)).collect();
+
+            rh.init(&token_hashes[0..self.window_size]);
+            let start_line = significant_tokens[0].line;
+            let end_line = significant_tokens[self.window_size - 1].line;
+            global_hash_map.entry(rh.get()).or_default().push(CloneLocation {
+                file: file_path.clone(),
+                start_line,
+                end_line,
+            });
+
+            for i in self.window_size..token_hashes.len() {
+                let hash = rh.roll(token_hashes[i - self.window_size], token_hashes[i]);
+                let start_line = significant_tokens[i - self.window_size + 1].line;
+                let end_line = significant_tokens[i].line;
+                global_hash_map.entry(hash).or_default().push(CloneLocation {
+                    file: file_path.clone(),
+                    start_line,
+                    end_line,
+                });
+            }
+        }
+
+        let mut clones = Vec::new();
+        let mut clone_id = 0;
+
+        for (hash, mut locations) in global_hash_map {
+            if locations.len() > 1 {
+                locations.sort_by(|a, b| {
+                    a.file
+                        .cmp(&b.file)
+                        .then(a.start_line.cmp(&b.start_line))
+                        .then(a.end_line.cmp(&b.end_line))
+                });
+                locations.dedup();
+
+                if locations.len() > 1 {
+                    clone_id += 1;
+                    clones.push(Clone { id: clone_id, length: self.window_size, locations, hash });
+                }
+            }
+        }
+
+        clones.sort_by(|a, b| b.locations.len().cmp(&a.locations.len()));
+        Ok(clones)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_no_clones_in_simple_file() {
+        let source = r#"
+fn simple() {
+    let x = 5;
+    let y = 10;
+    return x + y;
+}
+"#;
+        let detector = CloneDetector::new(10);
+        let clones = detector
+            .detect_in_file(source, Language::Rust, PathBuf::from("test.rs"))
+            .unwrap();
+
+        assert_eq!(clones.len(), 0);
+    }
+
+    #[test]
+    fn test_detect_simple_clone() {
+        let source = r#"
+fn process_a() {
+    let x = input.get();
+    let y = x * 2;
+    let z = y + 5;
+    return z;
+}
+
+fn process_b() {
+    let x = input.get();
+    let y = x * 2;
+    let z = y + 5;
+    return z;
+}
+"#;
+        let detector = CloneDetector::new(5);
+        let clones = detector
+            .detect_in_file(source, Language::Rust, PathBuf::from("test.rs"))
+            .unwrap();
+
+        assert!(!clones.is_empty());
+
+        for clone in &clones {
+            assert!(clone.locations.len() >= 2);
+        }
+    }
+
+    #[test]
+    fn test_across_files() {
+        let file1 = r#"
+fn helper() {
+    for i in 0..10 {
+        println!("{}", i);
+    }
+}
+"#;
+        let file2 = r#"
+fn another() {
+    for i in 0..10 {
+        println!("{}", i);
+    }
+}
+"#;
+
+        let files = vec![
+            (PathBuf::from("file1.rs"), file1.to_string(), Language::Rust),
+            (PathBuf::from("file2.rs"), file2.to_string(), Language::Rust),
+        ];
+
+        let detector = CloneDetector::new(5);
+        let clones = detector.detect_across_files(&files).unwrap();
+
+        if !clones.is_empty() {
+            let has_cross_file = clones.iter().any(|clone| {
+                let files: std::collections::HashSet<_> = clone.locations.iter().map(|l| &l.file).collect();
+                files.len() > 1
+            });
+            assert!(has_cross_file, "Should detect clones across different files");
+        }
+    }
+
+    #[test]
+    fn test_min_tokens_threshold() {
+        let source = "let x = 5; let y = 10; let x = 5; let y = 10;";
+
+        let detector1 = CloneDetector::new(3);
+        let clones1 = detector1
+            .detect_in_file(source, Language::Rust, PathBuf::from("test.rs"))
+            .unwrap();
+
+        let detector2 = CloneDetector::new(100);
+        let clones2 = detector2
+            .detect_in_file(source, Language::Rust, PathBuf::from("test.rs"))
+            .unwrap();
+
+        assert!(clones2.len() <= clones1.len());
+    }
+}

crates/core/src/cloner/mod.rs

···
+pub mod detector;
+pub mod rolling_hash;
+
+pub use detector::{Clone, CloneDetector, CloneLocation};
+pub use rolling_hash::RollingHash;

crates/core/src/cloner/rolling_hash.rs

···
+/// Rabin-Karp rolling hash implementation for efficient string/token matching
+///
+/// This uses polynomial rolling hash with a large prime modulus to minimize collisions
+/// while allowing O(1) hash updates when the window slides.
+pub struct RollingHash {
+    /// Base for polynomial hash (using 257 for byte values + 1)
+    base: u64,
+    /// Large prime modulus to reduce collisions
+    modulus: u64,
+    /// Current hash value
+    hash: u64,
+    /// Window size
+    window_size: usize,
+    /// Precomputed base^(window_size-1) mod modulus for removing leftmost element
+    base_power: u64,
+}
+
+impl RollingHash {
+    /// Create a new rolling hash with the specified window size
+    pub fn new(window_size: usize) -> Self {
+        let base = 257u64;
+        let modulus = 1_000_000_007u64;
+
+        let mut base_power = 1u64;
+        for _ in 0..window_size.saturating_sub(1) {
+            base_power = Self::mul_mod(base_power, base, modulus);
+        }
+
+        Self { base, modulus, hash: 0, window_size, base_power }
+    }
+
+    /// Initialize hash with a sequence of values
+    pub fn init(&mut self, values: &[u64]) {
+        self.hash = 0;
+        for &val in values.iter().take(self.window_size) {
+            self.hash = Self::mul_mod(self.hash, self.base, self.modulus);
+            self.hash = Self::add_mod(self.hash, val, self.modulus);
+        }
+    }
+
+    /// Roll the window: remove the leftmost value and add a new rightmost value
+    /// Returns the new hash value
+    pub fn roll(&mut self, old_value: u64, new_value: u64) -> u64 {
+        let old_contrib = Self::mul_mod(old_value, self.base_power, self.modulus);
+        self.hash = Self::sub_mod(self.hash, old_contrib, self.modulus);
+
+        self.hash = Self::mul_mod(self.hash, self.base, self.modulus);
+        self.hash = Self::add_mod(self.hash, new_value, self.modulus);
+
+        self.hash
+    }
+
+    /// Get current hash value
+    pub fn get(&self) -> u64 {
+        self.hash
+    }
+
+    /// Multiply with modular arithmetic to prevent overflow
+    fn mul_mod(a: u64, b: u64, modulus: u64) -> u64 {
+        ((a as u128 * b as u128) % modulus as u128) as u64
+    }
+
+    /// Add with modular arithmetic
+    fn add_mod(a: u64, b: u64, modulus: u64) -> u64 {
+        ((a as u128 + b as u128) % modulus as u128) as u64
+    }
+
+    /// Subtract with modular arithmetic (handles underflow)
+    fn sub_mod(a: u64, b: u64, modulus: u64) -> u64 {
+        if a >= b { (a - b) % modulus } else { (modulus - (b - a) % modulus) % modulus }
+    }
+}
+
+/// Compute a hash value for a token string (for use in rolling hash)
+pub fn token_hash(token: &str) -> u64 {
+    let mut hash = 5381u64;
+    for byte in token.bytes() {
+        hash = hash.wrapping_mul(33).wrapping_add(byte as u64);
+    }
+    hash
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_rolling_hash_basic() {
+        let window_size = 3;
+        let mut rh = RollingHash::new(window_size);
+
+        let values = [1, 2, 3, 4, 5];
+        rh.init(&values[0..3]);
+
+        let hash1 = rh.get();
+        assert_ne!(hash1, 0);
+
+        let hash2 = rh.roll(1, 4);
+        assert_ne!(hash2, 0);
+        assert_ne!(hash1, hash2);
+
+        let hash3 = rh.roll(2, 5);
+        assert_ne!(hash3, 0);
+        assert_ne!(hash2, hash3);
+    }
+
+    #[test]
+    fn test_same_sequence_same_hash() {
+        let mut rh1 = RollingHash::new(4);
+        let mut rh2 = RollingHash::new(4);
+
+        let seq = vec![10, 20, 30, 40];
+        rh1.init(&seq);
+        rh2.init(&seq);
+
+        assert_eq!(rh1.get(), rh2.get());
+    }
+
+    #[test]
+    fn test_different_sequences_different_hash() {
+        let mut rh1 = RollingHash::new(4);
+        let mut rh2 = RollingHash::new(4);
+
+        rh1.init(&[1, 2, 3, 4]);
+        rh2.init(&[1, 2, 3, 5]);
+
+        assert_ne!(rh1.get(), rh2.get());
+    }
+
+    #[test]
+    fn test_token_hash() {
+        let h1 = token_hash("if");
+        let h2 = token_hash("else");
+        let h3 = token_hash("if");
+
+        assert_ne!(h1, h2);
+        assert_eq!(h1, h3);
+    }
+
+    #[test]
+    fn test_rolling_preserves_pattern() {
+        let mut rh = RollingHash::new(3);
+        let values = vec![1, 2, 3, 4, 5, 6, 1, 2, 3];
+
+        rh.init(&values[0..3]);
+        let first_hash = rh.get();
+
+        let mut current_hash = first_hash;
+        for i in 3..9 {
+            current_hash = rh.roll(values[i - 3], values[i]);
+        }
+
+        assert_eq!(current_hash, first_hash);
+    }
+}