<s> import unittest <EOL> from random import randint , choice , shuffle , sample <EOL> from time import clock as now <EOL> from whoosh import fields , index , qparser , query , searching , scoring <EOL> from whoosh . filedb . filestore import RamStorage <EOL> from whoosh . postings import EmptyScorer , FakeScorer , AndNotScorer , UnionScorer , InverseScorer <EOL> from whoosh . query import * <EOL> class TestScorers ( unittest . TestCase ) : <EOL> def _keys ( self , searcher , docnums ) : <EOL> return sorted ( [ searcher . stored_fields ( docnum ) [ '<STR_LIT:key>' ] for docnum in docnums ] ) <EOL> def test_intersection ( self ) : <EOL> schema = fields . Schema ( key = fields . ID ( stored = True ) , value = fields . TEXT ( stored = True ) ) <EOL> st = RamStorage ( ) <EOL> ix = st . create_index ( schema ) <EOL> w = ix . writer ( ) <EOL> w . add_document ( key = u"<STR_LIT:a>" , value = u"<STR_LIT>" ) <EOL> w . add_document ( key = u"<STR_LIT:b>" , value = u"<STR_LIT>" ) <EOL> w . add_document ( key = u"<STR_LIT:c>" , value = u"<STR_LIT>" ) <EOL> w . commit ( ) <EOL> w = ix . writer ( ) <EOL> w . add_document ( key = u"<STR_LIT:d>" , value = u"<STR_LIT>" ) <EOL> w . add_document ( key = u"<STR_LIT:e>" , value = u"<STR_LIT>" ) <EOL> w . commit ( ) <EOL> searcher = ix . searcher ( ) <EOL> q = And ( [ Term ( "<STR_LIT:value>" , u"<STR_LIT>" ) , Term ( "<STR_LIT:value>" , u"<STR_LIT>" ) ] ) <EOL> sc = q . scorer ( searcher ) <EOL> self . assertEqual ( self . _keys ( searcher , sc . all_ids ( ) ) , [ "<STR_LIT:a>" , "<STR_LIT:e>" ] ) <EOL> q = And ( [ Term ( "<STR_LIT:value>" , u"<STR_LIT>" ) , Term ( "<STR_LIT:value>" , u"<STR_LIT>" ) ] ) <EOL> sc = q . scorer ( searcher ) <EOL> self . assertEqual ( self . _keys ( searcher , sc . all_ids ( ) ) , [ "<STR_LIT:a>" , "<STR_LIT:b>" , "<STR_LIT:d>" ] ) <EOL> def test_random_intersections ( self ) : <EOL> vals = [ u"<STR_LIT>" , u"<STR_LIT>" , u"<STR_LIT>" , u"<STR_LIT>" , u"<STR_LIT>" , u"<STR_LIT>" , u"<STR_LIT>" , <EOL> u"<STR_LIT>" , u"<STR_LIT>" , u"<STR_LIT>" , u"<STR_LIT>" , u"<STR_LIT>" , u"<STR_LIT>" ] <EOL> segments = <NUM_LIT:5> <EOL> docsperseg = <NUM_LIT:50> <EOL> fieldlimits = ( <NUM_LIT:3> , <NUM_LIT:10> ) <EOL> documents = [ ] <EOL> schema = fields . Schema ( key = fields . ID ( stored = True ) , value = fields . TEXT ( stored = True ) ) <EOL> st = RamStorage ( ) <EOL> ix = st . create_index ( schema ) <EOL> docnum = <NUM_LIT:0> <EOL> for segnum in xrange ( segments ) : <EOL> w = ix . writer ( ) <EOL> for d in xrange ( docsperseg ) : <EOL> doc = u"<STR_LIT:U+0020>" . join ( choice ( vals ) for _ in xrange ( randint ( * fieldlimits ) ) ) <EOL> w . add_document ( key = unicode ( docnum ) , value = doc ) <EOL> documents . append ( ( str ( docnum ) , doc ) ) <EOL> docnum += <NUM_LIT:1> <EOL> w . commit ( ) <EOL> self . assertNotEqual ( len ( ix . segments ) , <NUM_LIT:1> ) <EOL> testcount = <NUM_LIT:50> <EOL> testlimits = ( <NUM_LIT:2> , <NUM_LIT:5> ) <EOL> searcher = ix . searcher ( ) <EOL> for testnum in xrange ( testcount ) : <EOL> matches = [ ] <EOL> while not matches : <EOL> targets = sample ( vals , randint ( * testlimits ) ) <EOL> for docnum , doc in documents : <EOL> if all ( ( doc . find ( target ) > - <NUM_LIT:1> ) for target in targets ) : <EOL> matches . append ( docnum ) <EOL> matches . sort ( ) <EOL> q = And ( [ Term ( "<STR_LIT:value>" , target ) for target in targets ] ) <EOL> sc = q . scorer ( searcher ) <EOL> ids1 = list ( sc . all_ids ( ) ) <EOL> sc . reset ( ) <EOL> ids2 = [ ] <EOL> while sc . id is not None : <EOL> ids2 . append ( sc . id ) <EOL> sc . next ( ) <EOL> self . assertEqual ( ids1 , ids2 ) <EOL> keys = self . _keys ( searcher , ids1 ) <EOL> self . assertEqual ( keys , matches ) <EOL> def test_union ( self ) : <EOL> s1 = FakeScorer ( <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> ) <EOL> s2 = FakeScorer ( <NUM_LIT:2> , <NUM_LIT:4> , <NUM_LIT:8> , <NUM_LIT:10> , <NUM_LIT:20> , <NUM_LIT:30> ) <EOL> s3 = FakeScorer ( <NUM_LIT:10> , <NUM_LIT:100> , <NUM_LIT:200> ) <EOL> result = [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:10> , <NUM_LIT:20> , <NUM_LIT:30> , <NUM_LIT:100> , <NUM_LIT:200> ] <EOL> uqs = UnionScorer ( [ s1 , s2 , s3 ] ) <EOL> self . assertEqual ( list ( uqs . ids ( ) ) , result ) <EOL> def test_union_scores ( self ) : <EOL> s1 = FakeScorer ( <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ) <EOL> s2 = FakeScorer ( <NUM_LIT:2> , <NUM_LIT:4> , <NUM_LIT:8> ) <EOL> s3 = FakeScorer ( <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:8> ) <EOL> result = [ ( <NUM_LIT:1> , <NUM_LIT:10> ) , ( <NUM_LIT:2> , <NUM_LIT:30> ) , ( <NUM_LIT:3> , <NUM_LIT:20> ) , ( <NUM_LIT:4> , <NUM_LIT:10> ) , ( <NUM_LIT:8> , <NUM_LIT:20> ) ] <EOL> uqs = UnionScorer ( [ s1 , s2 , s3 ] ) <EOL> self . assertEqual ( list ( uqs ) , result ) <EOL> def test_random_union ( self ) : <EOL> testcount = <NUM_LIT:1000> <EOL> rangelimits = ( <NUM_LIT:2> , <NUM_LIT:10> ) <EOL> clauselimits = ( <NUM_LIT:2> , <NUM_LIT:10> ) <EOL> vals = range ( <NUM_LIT:100> ) <EOL> for testnum in xrange ( testcount ) : <EOL> matches = set ( ) <EOL> scorers = [ ] <EOL> for _ in xrange ( randint ( * clauselimits ) ) : <EOL> nums = sample ( vals , randint ( * rangelimits ) ) <EOL> matches = matches . union ( nums ) <EOL> scorers . append ( FakeScorer ( * sorted ( nums ) ) ) <EOL> matches = sorted ( matches ) <EOL> uqs = UnionScorer ( scorers ) <EOL> self . assertEqual ( list ( uqs . ids ( ) ) , matches ) <EOL> def test_inverse ( self ) : <EOL> s = FakeScorer ( <NUM_LIT:1> , <NUM_LIT:5> , <NUM_LIT:10> , <NUM_LIT:11> , <NUM_LIT> ) <EOL> inv = InverseScorer ( s , <NUM_LIT:15> , lambda id : False ) <EOL> scores = [ ] <EOL> while inv . id is not None : <EOL> scores . append ( inv . id ) <EOL> inv . next ( ) <EOL> self . assertEqual ( scores , [ <NUM_LIT:0> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> , <NUM_LIT:12> , <NUM_LIT> ] ) <EOL> def test_inverse_skip ( self ) : <EOL> s = FakeScorer ( <NUM_LIT:1> , <NUM_LIT:5> , <NUM_LIT:10> , <NUM_LIT:11> , <NUM_LIT> ) <EOL> inv = InverseScorer ( s , <NUM_LIT:15> , lambda id : False ) <EOL> inv . skip_to ( <NUM_LIT:8> ) <EOL> scores = [ ] <EOL> while inv . id is not None : <EOL> scores . append ( inv . id ) <EOL> inv . next ( ) <EOL> self . assertEqual ( scores , [ <NUM_LIT:8> , <NUM_LIT:9> , <NUM_LIT:12> , <NUM_LIT> ] ) <EOL> def test_andnot ( self ) : <EOL> pos = FakeScorer ( <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> , <NUM_LIT:10> ) <EOL> neg = FakeScorer ( <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:5> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:10> ) <EOL> ans = AndNotScorer ( pos , neg ) <EOL> ids = list ( ans . all_ids ( ) ) <EOL> self . assertEqual ( ids , [ <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:6> , <NUM_LIT:9> ] ) <EOL> def test_empty_andnot ( self ) : <EOL> pos = EmptyScorer ( ) <EOL> neg = EmptyScorer ( ) <EOL> ans = AndNotScorer ( pos , neg ) <EOL> ids = list ( ans . all_ids ( ) ) <EOL> self . assertEqual ( ids , [ ] ) <EOL> pos = FakeScorer ( <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> , <NUM_LIT:10> ) <EOL> neg = EmptyScorer ( ) <EOL> ans = AndNotScorer ( pos , neg ) <EOL> ids = list ( ans . all_ids ( ) ) <EOL> self . assertEqual ( ids , [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> , <NUM_LIT:10> ] ) <EOL> def test_random_andnot ( self ) : <EOL> testcount = <NUM_LIT:100> <EOL> rangesize = <NUM_LIT:100> <EOL> rng = range ( rangesize ) <EOL> for testnum in xrange ( testcount ) : <EOL> negs = sorted ( sample ( rng , randint ( <NUM_LIT:0> , rangesize - <NUM_LIT:1> ) ) ) <EOL> negset = frozenset ( negs ) <EOL> matched = [ n for n in rng if n not in negset ] <EOL> pos = FakeScorer ( * rng ) <EOL> neg = FakeScorer ( * negs ) <EOL> ans = AndNotScorer ( pos , neg ) <EOL> ids = list ( ans . all_ids ( ) ) <EOL> self . assertEqual ( ids , matched ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> unittest . main ( ) </s>
