<s> from __future__ import absolute_import , unicode_literals <EOL> import re <EOL> import unittest <EOL> import numpy as np <EOL> from spacy import attrs <EOL> from textacy import data , extract , preprocess , regexes_etc <EOL> class ExtractTestCase ( unittest . TestCase ) : <EOL> def setUp ( self ) : <EOL> self . maxDiff = None <EOL> spacy_pipeline = data . load_spacy_pipeline ( lang = '<STR_LIT>' ) <EOL> text = """<STR_LIT>""" <EOL> self . spacy_doc = spacy_pipeline ( text . strip ( ) ) <EOL> cols = [ attrs . TAG , attrs . HEAD , attrs . DEP ] <EOL> values = np . array ( <EOL> [ [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:11> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:5> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:4> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:12> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:6> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:5> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:4> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:4> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:4> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:4> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:4> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:4> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:9> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:10> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:11> , <NUM_LIT> ] ] , dtype = '<STR_LIT>' ) <EOL> self . spacy_doc . from_array ( cols , values ) <EOL> def test_words ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:U+002C>' , '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:.>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:U+002C>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ tok . orth_ for tok in extract . words ( <EOL> self . spacy_doc , filter_stops = False , filter_punct = False , filter_nums = False ) ] [ : <NUM_LIT> ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_words_filter ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ tok . orth_ for tok in extract . words ( <EOL> self . spacy_doc , filter_stops = True , filter_punct = True , filter_nums = True ) ] [ : <NUM_LIT> ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_words_good_tags ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:I>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ tok . orth_ for tok in extract . words ( <EOL> self . spacy_doc , filter_stops = False , filter_punct = False , filter_nums = False , <EOL> good_pos_tags = { '<STR_LIT>' } ) ] [ : <NUM_LIT> ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_words_min_freq ( self ) : <EOL> expected = [ <EOL> '<STR_LIT:U+002C>' , '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:.>' , '<STR_LIT>' , '<STR_LIT:U+002C>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:.>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:">' , '<STR_LIT:I>' ] <EOL> observed = [ tok . orth_ for tok in extract . words ( <EOL> self . spacy_doc , filter_stops = False , filter_punct = False , filter_nums = False , <EOL> min_freq = <NUM_LIT:2> ) ] [ : <NUM_LIT> ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_ngrams_less_than_1 ( self ) : <EOL> with self . assertRaises ( ValueError ) : <EOL> list ( extract . ngrams ( self . spacy_doc , <NUM_LIT:0> ) ) <EOL> def test_ngrams_1 ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:U+002C>' , '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:.>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:U+002C>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ span . orth_ for span in extract . ngrams ( <EOL> self . spacy_doc , <NUM_LIT:1> , filter_stops = False , filter_punct = False , filter_nums = False ) ] [ : <NUM_LIT> ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_ngrams_2 ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] <EOL> observed = [ span . orth_ for span in extract . ngrams ( <EOL> self . spacy_doc , <NUM_LIT:2> , filter_stops = False , filter_punct = False , filter_nums = False ) ] [ : <NUM_LIT> ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_ngrams_filter ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ span . orth_ for span in extract . ngrams ( <EOL> self . spacy_doc , <NUM_LIT:2> , filter_stops = True , filter_punct = True , filter_nums = True ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_ngrams_min_freq ( self ) : <EOL> expected = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ span . orth_ for span in extract . ngrams ( <EOL> self . spacy_doc , <NUM_LIT:2> , filter_stops = False , filter_punct = False , filter_nums = False , <EOL> min_freq = <NUM_LIT:2> ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_ngrams_good_tag ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ span . orth_ for span in extract . ngrams ( <EOL> self . spacy_doc , <NUM_LIT:2> , filter_stops = False , filter_punct = False , filter_nums = False , <EOL> good_pos_tags = { '<STR_LIT>' } ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_named_entities ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ ent . text for ent in extract . named_entities ( <EOL> self . spacy_doc , drop_determiners = True ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_named_entities_good ( self ) : <EOL> expected = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ ent . text for ent in extract . named_entities ( <EOL> self . spacy_doc , good_ne_types = { '<STR_LIT>' , '<STR_LIT>' } , drop_determiners = True ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_named_entities_min_freq ( self ) : <EOL> expected = [ ] <EOL> observed = [ ent . text for ent in extract . named_entities ( <EOL> self . spacy_doc , drop_determiners = True , min_freq = <NUM_LIT:2> ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_named_entities_determiner ( self ) : <EOL> expected = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ ent . text for ent in extract . named_entities ( <EOL> self . spacy_doc , drop_determiners = False ) if ent [ <NUM_LIT:0> ] . pos_ == '<STR_LIT>' ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_noun_chunks ( self ) : <EOL> expected = [ <EOL> '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ nc . text for nc in extract . noun_chunks ( <EOL> self . spacy_doc , drop_determiners = True ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_noun_chunks_determiner ( self ) : <EOL> expected = [ <EOL> '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ nc . text for nc in extract . noun_chunks ( <EOL> self . spacy_doc , drop_determiners = False ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_noun_chunks_min_freq ( self ) : <EOL> expected = [ '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ nc . text for nc in extract . noun_chunks ( <EOL> self . spacy_doc , drop_determiners = True , min_freq = <NUM_LIT:2> ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_pos_regex_matches ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT:I>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ span . text for span in extract . pos_regex_matches ( <EOL> self . spacy_doc , regexes_etc . POS_REGEX_PATTERNS [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_subject_verb_object_triples ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] <EOL> observed = [ '<STR_LIT:U+002CU+0020>' . join ( item . text for item in triple ) for triple in <EOL> extract . subject_verb_object_triples ( self . spacy_doc ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_acronyms_and_definitions ( self ) : <EOL> expected = { '<STR_LIT>' : '<STR_LIT>' } <EOL> observed = extract . acronyms_and_definitions ( self . spacy_doc ) <EOL> self . assertEqual ( observed , expected ) <EOL> def test_direct_quotations ( self ) : <EOL> expected = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ] <EOL> observed = [ '<STR_LIT:U+002CU+0020>' . join ( item . text for item in triple ) for triple in <EOL> extract . direct_quotations ( self . spacy_doc ) ] <EOL> self . assertEqual ( observed , expected ) </s>
