From 792515fb0d392e416e79c52081fd2e125dee5c47 Mon Sep 17 00:00:00 2001 From: Dima Kudosh Date: Sat, 27 Feb 2016 21:55:43 +0300 Subject: [PATCH] Change algorithm for finding largest common sequence, change SequenceMatcher to generic --- src/differ.rs | 55 +++++----- src/lib.rs | 43 +++++++- src/sequencematcher.rs | 224 +++++++++++++++++++++++++++-------------- src/utils.rs | 29 +++--- 4 files changed, 232 insertions(+), 119 deletions(-) diff --git a/src/differ.rs b/src/differ.rs index c89d6fe..ee016a0 100644 --- a/src/differ.rs +++ b/src/differ.rs @@ -1,5 +1,5 @@ -use sequencematcher::SequenceMatcher; -use utils::{slice_str, str_with_similar_chars, count_leading}; +use sequencematcher::{SequenceMatcher, Sequence}; +use utils::{str_with_similar_chars, count_leading}; use std::cmp; @@ -16,7 +16,7 @@ impl Differ{ } } - pub fn compare(&self, first_sequence: &str, second_sequence: &str) -> Vec> { + pub fn compare(&self, first_sequence: &T, second_sequence: &T) -> Vec> { let mut matcher = SequenceMatcher::new(first_sequence, second_sequence); let mut res = Vec::new(); for opcode in matcher.get_opcodes(){ @@ -32,21 +32,19 @@ impl Differ{ res } - fn dump(&self, tag: &str, string: &str, start: usize, end: usize) -> Vec { + fn dump(&self, tag: &str, sequence: &T, start: usize, end: usize) -> Vec { let mut res = Vec::new(); - match slice_str(string, start, end) { - Some(s) => { - for c in s.chars(){ - res.push(format!("{} {}", tag, c)); - } - }, - None => {}, - } + for i in start..end { + match sequence.at_index(i) { + Some(s) => res.push(format!("{} {}", tag, s)), + None => {}, + } + } res } - fn plain_replace(&self, first_sequence: &str, first_start: usize, first_end: usize, - second_sequence: &str, second_start: usize, second_end: usize) -> Vec { + fn plain_replace(&self, first_sequence: &T, first_start: usize, first_end: usize, + second_sequence: &T, second_start: usize, second_end: usize) -> Vec { if !(first_start < first_end && second_start < second_end){ return Vec::new(); } @@ -64,30 +62,28 @@ impl Differ{ } first } - - fn fancy_replace(&self, first_sequence: &str, first_start: usize, first_end: usize, - second_sequence: &str, second_start: usize, second_end: usize) -> Vec { + + fn fancy_replace(&self, first_sequence: &T, first_start: usize, first_end: usize, + second_sequence: &T, second_start: usize, second_end: usize) -> Vec { let mut res = Vec::new(); - let first_sequence_vec: Vec = first_sequence.chars().collect(); - let second_sequence_vec: Vec = second_sequence.chars().collect(); let (mut best_ratio, cutoff) = (0.74, 0.75); let (mut best_i, mut best_j) = (0, 0); - let (mut second_sequence_char, mut first_sequence_char) = (String::new(), String::new()); + let (mut second_sequence_str, mut first_sequence_str) = ("", ""); //cruncher.charjunk = self.charjunk; let mut eqi: Option = None; let mut eqj: Option = None; for j in second_start..second_end{ - second_sequence_char = second_sequence_vec[j].to_string(); + second_sequence_str = second_sequence.at_index(j).unwrap(); // Fix in feature for i in first_start..first_end{ - first_sequence_char = first_sequence_vec[i].to_string(); - if first_sequence_char == second_sequence_char{ + first_sequence_str = first_sequence.at_index(i).unwrap(); + if first_sequence_str == second_sequence_str{ if eqi.is_none(){ eqi = Some(i); eqj = Some(j); } continue; } - let mut cruncher = SequenceMatcher::new(&first_sequence_char, &second_sequence_char); + let mut cruncher = SequenceMatcher::new(first_sequence_str, second_sequence_str); if cruncher.ratio() > best_ratio{ best_ratio = cruncher.ratio(); best_i = i; @@ -107,10 +103,10 @@ impl Differ{ eqi = None; } res.extend(self.fancy_helper(first_sequence, first_start, best_i, second_sequence, second_start, best_j).iter().cloned()); - let (first_elt, second_elt) = (first_sequence_vec[best_i].to_string(), second_sequence_vec[best_j].to_string()); + let (first_elt, second_elt) = (first_sequence_str.at_index(best_i).unwrap(), second_sequence_str.at_index(best_j).unwrap()); if eqi.is_none(){ let (mut first_tag, mut second_tag) = (String::new(), String::new()); - let mut cruncher = SequenceMatcher::new(&first_elt, &second_elt); + let mut cruncher = SequenceMatcher::new(first_elt, second_elt); for opcode in &cruncher.get_opcodes(){ let (first_length, second_length) = (opcode.first_end - opcode.first_start, opcode.second_end - opcode.second_start); match opcode.tag.as_ref() { @@ -141,8 +137,8 @@ impl Differ{ res } - fn fancy_helper(&self, first_sequence: &str, first_start: usize, first_end: usize, - second_sequence: &str, second_start: usize, second_end: usize) -> Vec { + fn fancy_helper(&self, first_sequence: &T, first_start: usize, first_end: usize, + second_sequence: &T, second_start: usize, second_end: usize) -> Vec { let mut res = Vec::new(); if first_start < first_end{ if second_start < second_end{ @@ -175,4 +171,5 @@ impl Differ{ } res } -} \ No newline at end of file + +} diff --git a/src/lib.rs b/src/lib.rs index 7f9a8db..263a75a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ mod utils; use sequencematcher::SequenceMatcher; +use utils::{format_range_unified}; pub fn get_close_matches<'a>(word: &str, possibilities: Vec<&'a str>, n: usize, cutoff: f32) -> Vec<&'a str> { @@ -22,4 +23,44 @@ pub fn get_close_matches<'a>(word: &str, possibilities: Vec<&'a str>, n: usize, res.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap()); res.truncate(n); res.iter().map(|x| x.1).collect() -} \ No newline at end of file +} +/* +pub fn unified_diff(first_sequence: &str, second_sequence: &str, from_file: &str, to_file: &str, + from_file_date: &str, to_file_date: &str, n: usize, lineterm: char) -> Vec { + let mut res = Vec::new(); + let mut started = false; + let mut matcher = SequenceMatcher::new(first_sequence, second_sequence); + for group in &matcher.get_grouped_opcodes(n) { + if !started { + started = true; + let from_date = format!("\t{}", from_file_date); + let to_date = format!("\t{}", to_file_date); + res.push( format!("--- {}{}{}", from_file, from_date, lineterm) ); + res.push( format!("+++ {}{}{}", to_file, to_date, lineterm) ); + } + let (first, last) = (group.first().unwrap(), group.last().unwrap()); + let file1_range = format_range_unified(first.first_start, first.first_end); + let file2_range = format_range_unified(last.second_start, last.second_end); + res.push( format!("@@ -{} +{} @@{}", file1_range, file2_range, lineterm) ); + for code in group { + if code.tag == "equal" { + for line in slice_str(first_sequence, code.first_start, code.first_end).unwrap().split_whitespace() { + res.push( format!(" {}", line) ); + } + continue; + } + if code.tag == "replace" || code.tag == "delete" { + for line in slice_str(first_sequence, code.first_start, code.first_end).unwrap().split_whitespace() { + res.push( format!("-{}", line) ); + } + } + if code.tag == "replace" || code.tag == "insert" { + for line in slice_str(second_sequence, code.second_start, code.second_end).unwrap().split_whitespace() { + res.push( format!("+{}", line) ); + } + } + } + } + res +} +*/ \ No newline at end of file diff --git a/src/sequencematcher.rs b/src/sequencematcher.rs index f924f47..6af30ab 100644 --- a/src/sequencematcher.rs +++ b/src/sequencematcher.rs @@ -1,11 +1,12 @@ +use std::collections::HashMap; use std::cmp::{Ordering, max, min}; -use utils::{slice_str, calculate_ratio}; +use utils::calculate_ratio; #[derive(Debug, Clone, Copy)] pub struct Match{ - first_start: usize, - second_start: usize, - size: usize + pub first_start: usize, + pub second_start: usize, + pub size: usize } impl Ord for Match { @@ -59,75 +60,175 @@ impl Opcode{ } } +pub trait Sequence { + fn len(&self) -> usize; + fn at_index(&self, index: usize) -> Option<&str>; + fn slice(&self, start: usize, end: usize) -> Option<&str>; +} -pub struct SequenceMatcher<'a>{ - first_sequence: &'a str, - second_sequence: &'a str, - matching_blocks: Option>, - opcodes: Option>, - autojunk: bool +impl Sequence for str { + fn len(&self) -> usize { + self.len() + } + + fn slice(&self, start: usize, end: usize) -> Option<&str> { + if start >= end{ + return None + } + if start > self.len() || end > self.len(){ + return None + } + unsafe{ + Some(self.slice_unchecked(start, end)) + } + } + + fn at_index(&self, index: usize) -> Option<&str> { + self.slice(index, index + 1) + } +} + +impl<'a> Sequence for Vec<&'a str> { + fn len(&self) -> usize { + self.len() + } + + fn slice(&self, start: usize, end: usize) -> Option<&str> { + None + } + + fn at_index(&self, index: usize) -> Option<&str> { + if index < self.len() && index >= 0 { + return Some(self[index]) + } + None + } } -impl<'a> SequenceMatcher<'a>{ - pub fn new(first_sequence: &'a str, second_sequence: &'a str) -> SequenceMatcher<'a> { +pub struct SequenceMatcher<'a, T: 'a + ?Sized + Sequence>{ + first_sequence: &'a T, + second_sequence: &'a T, + matching_blocks: Option>, + opcodes: Option>, + autojunk: bool, + second_sequence_elements: HashMap<&'a str, Vec>, + second_sequence_popular: Vec<&'a str> +} + + +impl<'a, T: ?Sized + Sequence> SequenceMatcher<'a, T>{ + pub fn new(first_sequence: &'a T, second_sequence: &'a T) -> SequenceMatcher<'a, T> { + let mut matcher = SequenceMatcher{ first_sequence: first_sequence, second_sequence: second_sequence, matching_blocks: None, opcodes: None, autojunk: true, - - } + second_sequence_elements: HashMap::new(), + second_sequence_popular: Vec::new() + }; + matcher.set_seqs(first_sequence, second_sequence); + matcher } - pub fn set_seqs(&mut self, first_sequence: &'a str, second_sequence: &'a str) { + pub fn set_seqs(&mut self, first_sequence: &'a T, second_sequence: &'a T) { self.set_first_seq(first_sequence); self.set_second_seq(second_sequence); } - pub fn set_first_seq(&mut self, sequence: &'a str) { + pub fn set_first_seq(&mut self, sequence: &'a T) { self.first_sequence = sequence; self.matching_blocks = None; self.opcodes = None; } - pub fn set_second_seq(&mut self, sequence: &'a str) { + pub fn set_second_seq(&mut self, sequence: &'a T) { self.second_sequence = sequence; self.matching_blocks = None; self.opcodes = None; - //self.fullbcount = None - //self.__chain_b() + self.chain_second_seq(); } + + fn chain_second_seq(&mut self) { + let second_sequence = self.second_sequence; + let mut second_sequence_elements = HashMap::new(); + for i in 0..second_sequence.len() { + let mut counter = second_sequence_elements.entry(second_sequence.at_index(i).unwrap()).or_insert(Vec::new()); + counter.push(i); + } + let mut popular = Vec::new(); + let len = second_sequence.len(); + if len >= 200 { + let test_len = (len as f32 / 100.0).floor() as usize + 1; + for (element, indexes) in second_sequence_elements.iter() { + if indexes.len() > test_len { + popular.push(element.clone()); + } + } + for element in &popular { + second_sequence_elements.remove(element); + } + } + self.second_sequence_elements = second_sequence_elements; + self.second_sequence_popular = popular; + } - pub fn find_longest_match(&self, first_start: usize, first_end: usize, second_start: usize, second_end: usize) -> Option { - let first_sequence: Vec = slice_str(self.first_sequence, first_start, first_end).unwrap_or("").chars().collect(); - let second_sequence: Vec = slice_str(self.second_sequence, second_start, second_end).unwrap_or("").chars().collect(); - let mut max_i = 0; - let mut max_j = 0; - let mut arr: Vec> = vec![vec![0; second_sequence.len() + 1]; first_sequence.len() + 1]; - for i in 0..first_sequence.len(){ - for j in 0..second_sequence.len(){ - if first_sequence[i] == second_sequence[j]{ - arr[i + 1][j + 1] = arr[i][j] + 1; - if arr[i + 1][j + 1] > arr[max_i][max_j] { - max_i = i + 1; - max_j = j + 1; - } - } + pub fn find_longest_match(&self, first_start: usize, first_end: usize, second_start: usize, second_end: usize) -> Match { + let first_sequence = &self.first_sequence; + let second_sequence = &self.second_sequence; + let second_sequence_elements = &self.second_sequence_elements; + let (mut best_i, mut best_j, mut best_size) = (first_start, second_start, 0); + let mut j2len: HashMap = HashMap::new(); + for i in first_start..first_end { + let mut new_j2len: HashMap = HashMap::new(); + match second_sequence_elements.get(first_sequence.at_index(i).unwrap()) { + Some(indexes) => { + for j in indexes { + let j = j.clone(); + if j < second_start { + continue; + }; + if j >= second_end { + break; + }; + let mut size = 0; + if j > 0 { + match j2len.get(&(j-1)){ + Some(k) => { + size = k.clone(); + }, + None => {} + } + } + size += 1; + new_j2len.insert(j, size); + println!("{:?} {}", i, size); + if size > best_size { + best_i = i + 1 - size; + best_j = j + 1 - size; + best_size = size; + } + } + }, + None => {}, } + j2len = new_j2len; } - if max_i == 0 && max_j == 0{ - return None - } - else{ - let size = arr[max_i][max_j]; - Some(Match{ - first_start: max_i - size + first_start, - second_start: max_j - size + second_start, - size: size - }) - } + for _ in 0..2 { + while best_i > first_start && best_j > second_start && + first_sequence.at_index(best_i - 1) == second_sequence.at_index(best_j-1){ + best_i = best_i - 1; + best_j = best_j - 1; + best_size = best_size + 1; + } + while best_i + best_size < first_end && best_j + best_size < second_end && + first_sequence.at_index(best_i + best_size) == second_sequence.at_index(best_j + best_size) { + best_size += 1; + } + } + Match::new(best_i, best_j, best_size) } pub fn get_matching_blocks(&mut self) -> Vec { @@ -140,8 +241,9 @@ impl<'a> SequenceMatcher<'a>{ while !queue.is_empty(){ let (first_start, first_end, second_start, second_end) = queue.pop().unwrap(); let m = self.find_longest_match(first_start, first_end, second_start, second_end); - match m { - Some(m) => { + match m.size { + 0 => {}, + _ => { if first_start < m.first_start && second_start < m.second_start{ queue.push((first_start, m.first_start, second_start, m.second_start)); } @@ -150,9 +252,9 @@ impl<'a> SequenceMatcher<'a>{ } matches.push(m); }, - None => {}, } } + println!("{:?}", matches); matches.sort_by(|a, b| b.cmp(a)); let (mut first_start, mut second_start, mut size) = (0, 0, 0); let mut non_adjacent = Vec::new(); @@ -207,32 +309,6 @@ impl<'a> SequenceMatcher<'a>{ return self.opcodes.as_ref().unwrap().clone() } - /* - codes = self.get_opcodes() - if not codes: - codes = [("equal", 0, 1, 0, 1)] - # Fixup leading and trailing groups if they show no changes. - if codes[0][0] == 'equal': - tag, i1, i2, j1, j2 = codes[0] - codes[0] = tag, max(i1, i2-n), i2, max(j1, j2-n), j2 - if codes[-1][0] == 'equal': - tag, i1, i2, j1, j2 = codes[-1] - codes[-1] = tag, i1, min(i2, i1+n), j1, min(j2, j1+n) - - nn = n + n - group = [] - for tag, i1, i2, j1, j2 in codes: - # End the current group and start a new one whenever - # there is a large range with no changes. - if tag == 'equal' and i2-i1 > nn: - group.append((tag, i1, min(i2, i1+n), j1, min(j2, j1+n))) - yield group - group = [] - i1, j1 = max(i1, i2-n), max(j1, j2-n) - group.append((tag, i1, i2, j1 ,j2)) - if group and not (len(group)==1 and group[0][0] == 'equal'): - yield group - */ pub fn get_grouped_opcodes(&mut self, n: usize) -> Vec> { let mut res = Vec::new(); let mut codes = self.get_opcodes(); diff --git a/src/utils.rs b/src/utils.rs index 90c3efc..c4f7ea7 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,17 +1,3 @@ -pub fn slice_str(string: &str, start: usize, end: usize) -> Option<&str> -{ - if start >= end{ - return None - } - if start > string.len() || end > string.len(){ - return None - } - unsafe{ - Some(string.slice_unchecked(start, end)) - } -} - - pub fn calculate_ratio(matches: usize, length: usize) -> f32{ if length != 0{ return 2.0 * matches as f32 / length as f32 @@ -36,4 +22,17 @@ pub fn count_leading(line: &str, c: char) -> usize { i += 1; } return i -} \ No newline at end of file +} + + +pub fn format_range_unified(start:usize, end: usize) -> String { + let mut beginning = start + 1; + let length = end - start; + if length == 1 { + return format!("{}", beginning) + } + if length == 0 { + beginning -= 1; + } + format!("{},{}", beginning, length) +} \ No newline at end of file