/// Extract sequences of `length` words from all phrases and collect all those that repeat more than
/// once. For each sequence, it will regroup all the word locations that compose each instance of
/// the repeated sequence.
Extract sequences of `length` words from all phrases and collect all those that repeat more than
once. For each sequence, it will regroup all the word locations that compose each instance of
the repeated sequence.
Extract sequences of ` length ` words from all phrases and collect all those that repeat more than once . For each sequence it will regroup all the word locations that compose each instance of the repeated sequence .
fn extract_sequences_with_length(phrases: &[Phrase], length: usize) -> Vec<RepeatedSequence> {
    // Collect all sequences
    let mut sequences: BTreeMap<_, Vec<_>> = BTreeMap::new();
    for phrase in phrases {
        let max_end = phrase.words().len().saturating_sub(length - 1);
        for start_index in 0..max_end {
            let end_index = start_index + length;
            let word_tags = phrase.words()[start_index..end_index].iter().collect_vec();
            let locations = (start_index..end_index)
                .map(|index| PhrasedWordId {
                    phrase: phrase.id(),
                    word: WordId(index as u16),
                })
                .collect_vec();
            sequences.entry(word_tags).or_default().push(locations);
        }
    }
    // Select the sequences of interest
    sequences
        .into_iter()
        .filter_map(|(words, instances)| {
            if instances.len() == 1 {
                None
            } else {
                Some(RepeatedSequence { words, instances })
            }
        })
        .collect_vec()
}
fn extract_sequences_with_length ( phrases : & [ Phrase ] , length : usize ) -> Vec < RepeatedSequence > { // Collect all sequences let mut sequences : BTreeMap < _ , Vec < _ > > = BTreeMap :: new ( ) ; for phrase in phrases { let max_end = phrase . words ( ) . len ( ) . saturating_sub ( length - 1 ) ; for start_index in 0 .. max_end { let end_index = start_index + length ; let word_tags = phrase . words ( ) [ start_index .. end_index ] . iter ( ) . collect_vec ( ) ; let locations = ( start_index .. end_index ) . map ( | index | PhrasedWordId { phrase : phrase . id ( ) , word : WordId ( index as u16 ) , } ) . collect_vec ( ) ; sequences . entry ( word_tags ) . or_default ( ) . push ( locations ) ; } } // Select the sequences of interest sequences . into_iter ( ) . filter_map ( | ( words , instances ) | { if instances . len ( ) == 1 { None } else { Some ( RepeatedSequence { words , instances } ) } } ) . collect_vec ( ) }
Extract sequences of `length` words from all phrases and collect all those that repeat more than
once.
Extract sequences of ` length ` words from all phrases and collect all those that repeat more than once .
phrases
&[Phrase]
length
usize