<s> from __future__ import absolute_import , print_function , division <EOL> import itertools <EOL> import operator <EOL> from petl . compat import next , text_type <EOL> from petl . errors import ArgumentError <EOL> from petl . comparison import comparable_itemgetter , Comparable <EOL> from petl . util . base import Table , asindices , rowgetter , rowgroupby , header , data <EOL> from petl . transform . sorts import sort <EOL> from petl . transform . basics import cut , cutout <EOL> from petl . transform . dedup import distinct <EOL> def natural_key ( left , right ) : <EOL> lhdr = header ( left ) <EOL> lflds = list ( map ( str , lhdr ) ) <EOL> rhdr = header ( right ) <EOL> rflds = list ( map ( str , rhdr ) ) <EOL> key = [ f for f in lflds if f in rflds ] <EOL> assert len ( key ) > <NUM_LIT:0> , '<STR_LIT>' <EOL> if len ( key ) == <NUM_LIT:1> : <EOL> key = key [ <NUM_LIT:0> ] <EOL> return key <EOL> def keys_from_args ( left , right , key , lkey , rkey ) : <EOL> if key is lkey is rkey is None : <EOL> lkey = rkey = natural_key ( left , right ) <EOL> elif key is not None and lkey is rkey is None : <EOL> lkey = rkey = key <EOL> elif key is None and lkey is not None and rkey is not None : <EOL> pass <EOL> else : <EOL> raise ArgumentError ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> ) <EOL> return lkey , rkey <EOL> def join ( left , right , key = None , lkey = None , rkey = None , presorted = False , <EOL> buffersize = None , tempdir = None , cache = True , lprefix = None , rprefix = None ) : <EOL> """<STR_LIT>""" <EOL> lkey , rkey = keys_from_args ( left , right , key , lkey , rkey ) <EOL> return JoinView ( left , right , lkey = lkey , rkey = rkey , <EOL> presorted = presorted , buffersize = buffersize , tempdir = tempdir , <EOL> cache = cache , lprefix = lprefix , rprefix = rprefix ) <EOL> Table . join = join <EOL> class JoinView ( Table ) : <EOL> def __init__ ( self , left , right , lkey , rkey , <EOL> presorted = False , leftouter = False , rightouter = False , <EOL> missing = None , buffersize = None , tempdir = None , cache = True , <EOL> lprefix = None , rprefix = None ) : <EOL> self . lkey = lkey <EOL> self . rkey = rkey <EOL> if presorted : <EOL> self . left = left <EOL> self . right = right <EOL> else : <EOL> self . left = sort ( left , lkey , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache ) <EOL> self . right = sort ( right , rkey , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache ) <EOL> self . leftouter = leftouter <EOL> self . rightouter = rightouter <EOL> self . missing = missing <EOL> self . lprefix = lprefix <EOL> self . rprefix = rprefix <EOL> def __iter__ ( self ) : <EOL> return iterjoin ( self . left , self . right , self . lkey , self . rkey , <EOL> leftouter = self . leftouter , rightouter = self . rightouter , <EOL> missing = self . missing , lprefix = self . lprefix , <EOL> rprefix = self . rprefix ) <EOL> def leftjoin ( left , right , key = None , lkey = None , rkey = None , missing = None , <EOL> presorted = False , buffersize = None , tempdir = None , cache = True , <EOL> lprefix = None , rprefix = None ) : <EOL> """<STR_LIT>""" <EOL> lkey , rkey = keys_from_args ( left , right , key , lkey , rkey ) <EOL> return JoinView ( left , right , lkey = lkey , rkey = rkey , <EOL> presorted = presorted , leftouter = True , rightouter = False , <EOL> missing = missing , buffersize = buffersize , tempdir = tempdir , <EOL> cache = cache , lprefix = lprefix , rprefix = rprefix ) <EOL> Table . leftjoin = leftjoin <EOL> def rightjoin ( left , right , key = None , lkey = None , rkey = None , missing = None , <EOL> presorted = False , buffersize = None , tempdir = None , cache = True , <EOL> lprefix = None , rprefix = None ) : <EOL> """<STR_LIT>""" <EOL> lkey , rkey = keys_from_args ( left , right , key , lkey , rkey ) <EOL> return JoinView ( left , right , lkey = lkey , rkey = rkey , <EOL> presorted = presorted , leftouter = False , rightouter = True , <EOL> missing = missing , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache , lprefix = lprefix , <EOL> rprefix = rprefix ) <EOL> Table . rightjoin = rightjoin <EOL> def outerjoin ( left , right , key = None , lkey = None , rkey = None , missing = None , <EOL> presorted = False , buffersize = None , tempdir = None , cache = True , <EOL> lprefix = None , rprefix = None ) : <EOL> """<STR_LIT>""" <EOL> lkey , rkey = keys_from_args ( left , right , key , lkey , rkey ) <EOL> return JoinView ( left , right , lkey = lkey , rkey = rkey , <EOL> presorted = presorted , leftouter = True , rightouter = True , <EOL> missing = missing , buffersize = buffersize , tempdir = tempdir , <EOL> cache = cache , lprefix = lprefix , rprefix = rprefix ) <EOL> Table . outerjoin = outerjoin <EOL> def iterjoin ( left , right , lkey , rkey , leftouter = False , rightouter = False , <EOL> missing = None , lprefix = None , rprefix = None ) : <EOL> lit = iter ( left ) <EOL> rit = iter ( right ) <EOL> lhdr = next ( lit ) <EOL> rhdr = next ( rit ) <EOL> lkind = asindices ( lhdr , lkey ) <EOL> rkind = asindices ( rhdr , rkey ) <EOL> lgetk = comparable_itemgetter ( * lkind ) <EOL> rgetk = comparable_itemgetter ( * rkind ) <EOL> rvind = [ i for i in range ( len ( rhdr ) ) if i not in rkind ] <EOL> rgetv = rowgetter ( * rvind ) <EOL> if lprefix is None : <EOL> outhdr = list ( lhdr ) <EOL> else : <EOL> outhdr = [ ( text_type ( lprefix ) + text_type ( f ) ) for f in lhdr ] <EOL> if rprefix is None : <EOL> outhdr . extend ( rgetv ( rhdr ) ) <EOL> else : <EOL> outhdr . extend ( [ ( text_type ( rprefix ) + text_type ( f ) ) for f in rgetv ( rhdr ) ] ) <EOL> yield tuple ( outhdr ) <EOL> def joinrows ( _lrowgrp , _rrowgrp ) : <EOL> if _rrowgrp is None : <EOL> for lrow in _lrowgrp : <EOL> outrow = list ( lrow ) <EOL> outrow . extend ( [ missing ] * len ( rvind ) ) <EOL> yield tuple ( outrow ) <EOL> elif _lrowgrp is None : <EOL> for rrow in _rrowgrp : <EOL> outrow = [ missing ] * len ( lhdr ) <EOL> for li , ri in zip ( lkind , rkind ) : <EOL> outrow [ li ] = rrow [ ri ] <EOL> outrow . extend ( rgetv ( rrow ) ) <EOL> yield tuple ( outrow ) <EOL> else : <EOL> _rrowgrp = list ( _rrowgrp ) <EOL> for lrow in _lrowgrp : <EOL> for rrow in _rrowgrp : <EOL> outrow = list ( lrow ) <EOL> outrow . extend ( rgetv ( rrow ) ) <EOL> yield tuple ( outrow ) <EOL> lgit = itertools . groupby ( lit , key = lgetk ) <EOL> rgit = itertools . groupby ( rit , key = rgetk ) <EOL> lrowgrp = [ ] <EOL> rrowgrp = [ ] <EOL> lkval , rkval = Comparable ( None ) , Comparable ( None ) <EOL> try : <EOL> lkval , lrowgrp = next ( lgit ) <EOL> rkval , rrowgrp = next ( rgit ) <EOL> while True : <EOL> if lkval < rkval : <EOL> if leftouter : <EOL> for row in joinrows ( lrowgrp , None ) : <EOL> yield tuple ( row ) <EOL> lkval , lrowgrp = next ( lgit ) <EOL> elif lkval > rkval : <EOL> if rightouter : <EOL> for row in joinrows ( None , rrowgrp ) : <EOL> yield tuple ( row ) <EOL> rkval , rrowgrp = next ( rgit ) <EOL> else : <EOL> for row in joinrows ( lrowgrp , rrowgrp ) : <EOL> yield tuple ( row ) <EOL> lkval , lrowgrp = next ( lgit ) <EOL> rkval , rrowgrp = next ( rgit ) <EOL> except StopIteration : <EOL> pass <EOL> if leftouter : <EOL> if lkval > rkval : <EOL> for row in joinrows ( lrowgrp , None ) : <EOL> yield tuple ( row ) <EOL> for lkval , lrowgrp in lgit : <EOL> for row in joinrows ( lrowgrp , None ) : <EOL> yield tuple ( row ) <EOL> if rightouter : <EOL> if lkval < rkval : <EOL> for row in joinrows ( None , rrowgrp ) : <EOL> yield tuple ( row ) <EOL> for rkval , rrowgrp in rgit : <EOL> for row in joinrows ( None , rrowgrp ) : <EOL> yield tuple ( row ) <EOL> def crossjoin ( * tables , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return CrossJoinView ( * tables , ** kwargs ) <EOL> Table . crossjoin = crossjoin <EOL> class CrossJoinView ( Table ) : <EOL> def __init__ ( self , * sources , ** kwargs ) : <EOL> self . sources = sources <EOL> self . prefix = kwargs . get ( '<STR_LIT>' , False ) <EOL> def __iter__ ( self ) : <EOL> return itercrossjoin ( self . sources , self . prefix ) <EOL> def itercrossjoin ( sources , prefix ) : <EOL> outhdr = list ( ) <EOL> for i , s in enumerate ( sources ) : <EOL> if prefix : <EOL> outhdr . extend ( [ text_type ( i + <NUM_LIT:1> ) + '<STR_LIT:_>' + text_type ( f ) for f in header ( s ) ] ) <EOL> else : <EOL> outhdr . extend ( header ( s ) ) <EOL> yield tuple ( outhdr ) <EOL> datasrcs = [ data ( src ) for src in sources ] <EOL> for prod in itertools . product ( * datasrcs ) : <EOL> outrow = list ( ) <EOL> for row in prod : <EOL> outrow . extend ( row ) <EOL> yield tuple ( outrow ) <EOL> def antijoin ( left , right , key = None , lkey = None , rkey = None , presorted = False , <EOL> buffersize = None , tempdir = None , cache = True ) : <EOL> """<STR_LIT>""" <EOL> lkey , rkey = keys_from_args ( left , right , key , lkey , rkey ) <EOL> return AntiJoinView ( left = left , right = right , lkey = lkey , rkey = rkey , <EOL> presorted = presorted , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache ) <EOL> Table . antijoin = antijoin <EOL> class AntiJoinView ( Table ) : <EOL> def __init__ ( self , left , right , lkey , rkey , presorted = False , <EOL> buffersize = None , tempdir = None , cache = True ) : <EOL> if presorted : <EOL> self . left = left <EOL> self . right = right <EOL> else : <EOL> self . left = sort ( left , lkey , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache ) <EOL> self . right = sort ( right , rkey , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache ) <EOL> self . lkey = lkey <EOL> self . rkey = rkey <EOL> def __iter__ ( self ) : <EOL> return iterantijoin ( self . left , self . right , self . lkey , self . rkey ) <EOL> def iterantijoin ( left , right , lkey , rkey ) : <EOL> lit = iter ( left ) <EOL> rit = iter ( right ) <EOL> lhdr = next ( lit ) <EOL> rhdr = next ( rit ) <EOL> yield tuple ( lhdr ) <EOL> lkind = asindices ( lhdr , lkey ) <EOL> rkind = asindices ( rhdr , rkey ) <EOL> lgetk = comparable_itemgetter ( * lkind ) <EOL> rgetk = comparable_itemgetter ( * rkind ) <EOL> lgit = itertools . groupby ( lit , key = lgetk ) <EOL> rgit = itertools . groupby ( rit , key = rgetk ) <EOL> lrowgrp = [ ] <EOL> lkval , rkval = Comparable ( None ) , Comparable ( None ) <EOL> try : <EOL> lkval , lrowgrp = next ( lgit ) <EOL> rkval , _ = next ( rgit ) <EOL> while True : <EOL> if lkval < rkval : <EOL> for row in lrowgrp : <EOL> yield tuple ( row ) <EOL> lkval , lrowgrp = next ( lgit ) <EOL> elif lkval > rkval : <EOL> rkval , _ = next ( rgit ) <EOL> else : <EOL> lkval , lrowgrp = next ( lgit ) <EOL> rkval , _ = next ( rgit ) <EOL> except StopIteration : <EOL> pass <EOL> if lkval > rkval : <EOL> for row in lrowgrp : <EOL> yield tuple ( row ) <EOL> for lkval , lrowgrp in lgit : <EOL> for row in lrowgrp : <EOL> yield tuple ( row ) <EOL> def lookupjoin ( left , right , key = None , lkey = None , rkey = None , missing = None , <EOL> presorted = False , buffersize = None , tempdir = None , cache = True , <EOL> lprefix = None , rprefix = None ) : <EOL> """<STR_LIT>""" <EOL> lkey , rkey = keys_from_args ( left , right , key , lkey , rkey ) <EOL> return LookupJoinView ( left , right , lkey , rkey , presorted = presorted , <EOL> missing = missing , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache , <EOL> lprefix = lprefix , rprefix = rprefix ) <EOL> Table . lookupjoin = lookupjoin <EOL> class LookupJoinView ( Table ) : <EOL> def __init__ ( self , left , right , lkey , rkey , presorted = False , missing = None , <EOL> buffersize = None , tempdir = None , cache = True , <EOL> lprefix = None , rprefix = None ) : <EOL> if presorted : <EOL> self . left = left <EOL> self . right = right <EOL> else : <EOL> self . left = sort ( left , lkey , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache ) <EOL> self . right = sort ( right , rkey , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache ) <EOL> self . lkey = lkey <EOL> self . rkey = rkey <EOL> self . missing = missing <EOL> self . lprefix = lprefix <EOL> self . rprefix = rprefix <EOL> def __iter__ ( self ) : <EOL> return iterlookupjoin ( self . left , self . right , self . lkey , self . rkey , <EOL> missing = self . missing , lprefix = self . lprefix , <EOL> rprefix = self . rprefix ) <EOL> def iterlookupjoin ( left , right , lkey , rkey , missing = None , lprefix = None , <EOL> rprefix = None ) : <EOL> lit = iter ( left ) <EOL> rit = iter ( right ) <EOL> lhdr = next ( lit ) <EOL> rhdr = next ( rit ) <EOL> lkind = asindices ( lhdr , lkey ) <EOL> rkind = asindices ( rhdr , rkey ) <EOL> lgetk = operator . itemgetter ( * lkind ) <EOL> rgetk = operator . itemgetter ( * rkind ) <EOL> rvind = [ i for i in range ( len ( rhdr ) ) if i not in rkind ] <EOL> rgetv = rowgetter ( * rvind ) <EOL> if lprefix is None : <EOL> outhdr = list ( lhdr ) <EOL> else : <EOL> outhdr = [ ( text_type ( lprefix ) + text_type ( f ) ) for f in lhdr ] <EOL> if rprefix is None : <EOL> outhdr . extend ( rgetv ( rhdr ) ) <EOL> else : <EOL> outhdr . extend ( [ ( text_type ( rprefix ) + text_type ( f ) ) for f in rgetv ( rhdr ) ] ) <EOL> yield tuple ( outhdr ) <EOL> def joinrows ( _lrowgrp , _rrowgrp ) : <EOL> if _rrowgrp is None : <EOL> for lrow in _lrowgrp : <EOL> outrow = list ( lrow ) <EOL> outrow . extend ( [ missing ] * len ( rvind ) ) <EOL> yield tuple ( outrow ) <EOL> else : <EOL> rrow = next ( iter ( _rrowgrp ) ) <EOL> for lrow in _lrowgrp : <EOL> outrow = list ( lrow ) <EOL> outrow . extend ( rgetv ( rrow ) ) <EOL> yield tuple ( outrow ) <EOL> lgit = itertools . groupby ( lit , key = lgetk ) <EOL> rgit = itertools . groupby ( rit , key = rgetk ) <EOL> lrowgrp = [ ] <EOL> lkval , rkval = None , None <EOL> try : <EOL> lkval , lrowgrp = next ( lgit ) <EOL> rkval , rrowgrp = next ( rgit ) <EOL> while True : <EOL> if lkval < rkval : <EOL> for row in joinrows ( lrowgrp , None ) : <EOL> yield tuple ( row ) <EOL> lkval , lrowgrp = next ( lgit ) <EOL> elif lkval > rkval : <EOL> rkval , rrowgrp = next ( rgit ) <EOL> else : <EOL> for row in joinrows ( lrowgrp , rrowgrp ) : <EOL> yield tuple ( row ) <EOL> lkval , lrowgrp = next ( lgit ) <EOL> rkval , rrowgrp = next ( rgit ) <EOL> except StopIteration : <EOL> pass <EOL> if lkval > rkval : <EOL> for row in joinrows ( lrowgrp , None ) : <EOL> yield tuple ( row ) <EOL> for lkval , lrowgrp in lgit : <EOL> for row in joinrows ( lrowgrp , None ) : <EOL> yield tuple ( row ) <EOL> def unjoin ( table , value , key = None , autoincrement = ( <NUM_LIT:1> , <NUM_LIT:1> ) , presorted = False , <EOL> buffersize = None , tempdir = None , cache = True ) : <EOL> """<STR_LIT>""" <EOL> if key is None : <EOL> if presorted : <EOL> tbl_sorted = table <EOL> else : <EOL> tbl_sorted = sort ( table , value , buffersize = buffersize , <EOL> tempdir = tempdir , cache = cache ) <EOL> left = ConvertToIncrementingCounterView ( tbl_sorted , value , <EOL> autoincrement ) <EOL> right = EnumerateDistinctView ( tbl_sorted , value , autoincrement ) <EOL> else : <EOL> left = distinct ( cutout ( table , value ) ) <EOL> right = distinct ( cut ( table , key , value ) ) <EOL> return left , right <EOL> class ConvertToIncrementingCounterView ( Table ) : <EOL> def __init__ ( self , tbl , value , autoincrement ) : <EOL> self . table = tbl <EOL> self . value = value <EOL> self . autoincrement = autoincrement <EOL> def __iter__ ( self ) : <EOL> it = iter ( self . table ) <EOL> hdr = next ( it ) <EOL> table = itertools . chain ( [ hdr ] , it ) <EOL> value = self . value <EOL> vidx = hdr . index ( value ) <EOL> outhdr = list ( hdr ) <EOL> outhdr [ vidx ] = '<STR_LIT>' % value <EOL> yield tuple ( outhdr ) <EOL> offset , multiplier = self . autoincrement <EOL> for n , ( _ , group ) in enumerate ( rowgroupby ( table , value ) ) : <EOL> for row in group : <EOL> outrow = list ( row ) <EOL> outrow [ vidx ] = ( n * multiplier ) + offset <EOL> yield tuple ( outrow ) <EOL> Table . unjoin = unjoin <EOL> class EnumerateDistinctView ( Table ) : <EOL> def __init__ ( self , tbl , value , autoincrement ) : <EOL> self . table = tbl <EOL> self . value = value <EOL> self . autoincrement = autoincrement <EOL> def __iter__ ( self ) : <EOL> offset , multiplier = self . autoincrement <EOL> yield ( '<STR_LIT:id>' , self . value ) <EOL> for n , ( v , _ ) in enumerate ( rowgroupby ( self . table , self . value ) ) : <EOL> yield ( ( n * multiplier ) + offset , v ) </s>
