<s> import gzip <EOL> import io <EOL> import platform <EOL> import struct <EOL> import six <EOL> from six . moves import xrange <EOL> _XERIAL_V1_HEADER = ( - <NUM_LIT> , b'<STR_LIT:S>' , b'<STR_LIT:N>' , b'<STR_LIT:A>' , b'<STR_LIT:P>' , b'<STR_LIT:P>' , b'<STR_LIT:Y>' , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> _XERIAL_V1_FORMAT = '<STR_LIT>' <EOL> try : <EOL> import snappy <EOL> except ImportError : <EOL> snappy = None <EOL> try : <EOL> import lz4f <EOL> import xxhash <EOL> except ImportError : <EOL> lz4f = None <EOL> PYPY = bool ( platform . python_implementation ( ) == '<STR_LIT>' ) <EOL> def has_gzip ( ) : <EOL> return True <EOL> def has_snappy ( ) : <EOL> return snappy is not None <EOL> def has_lz4 ( ) : <EOL> return lz4f is not None <EOL> def gzip_encode ( payload , compresslevel = None ) : <EOL> if not compresslevel : <EOL> compresslevel = <NUM_LIT:9> <EOL> buf = io . BytesIO ( ) <EOL> gzipper = gzip . GzipFile ( fileobj = buf , mode = "<STR_LIT:w>" , compresslevel = compresslevel ) <EOL> try : <EOL> gzipper . write ( payload ) <EOL> finally : <EOL> gzipper . close ( ) <EOL> return buf . getvalue ( ) <EOL> def gzip_decode ( payload ) : <EOL> buf = io . BytesIO ( payload ) <EOL> gzipper = gzip . GzipFile ( fileobj = buf , mode = '<STR_LIT:r>' ) <EOL> try : <EOL> return gzipper . read ( ) <EOL> finally : <EOL> gzipper . close ( ) <EOL> def snappy_encode ( payload , xerial_compatible = True , xerial_blocksize = <NUM_LIT:32> * <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> if not has_snappy ( ) : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> if not xerial_compatible : <EOL> return snappy . compress ( payload ) <EOL> out = io . BytesIO ( ) <EOL> for fmt , dat in zip ( _XERIAL_V1_FORMAT , _XERIAL_V1_HEADER ) : <EOL> out . write ( struct . pack ( '<STR_LIT:!>' + fmt , dat ) ) <EOL> if PYPY : <EOL> chunker = lambda payload , i , size : payload [ i : size + i ] <EOL> elif six . PY2 : <EOL> chunker = lambda payload , i , size : buffer ( payload , i , size ) <EOL> else : <EOL> chunker = lambda payload , i , size : memoryview ( payload ) [ i : size + i ] . tobytes ( ) <EOL> for chunk in ( chunker ( payload , i , xerial_blocksize ) <EOL> for i in xrange ( <NUM_LIT:0> , len ( payload ) , xerial_blocksize ) ) : <EOL> block = snappy . compress ( chunk ) <EOL> block_size = len ( block ) <EOL> out . write ( struct . pack ( '<STR_LIT>' , block_size ) ) <EOL> out . write ( block ) <EOL> return out . getvalue ( ) <EOL> def _detect_xerial_stream ( payload ) : <EOL> """<STR_LIT>""" <EOL> if len ( payload ) > <NUM_LIT:16> : <EOL> header = struct . unpack ( '<STR_LIT:!>' + _XERIAL_V1_FORMAT , bytes ( payload ) [ : <NUM_LIT:16> ] ) <EOL> return header == _XERIAL_V1_HEADER <EOL> return False <EOL> def snappy_decode ( payload ) : <EOL> if not has_snappy ( ) : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> if _detect_xerial_stream ( payload ) : <EOL> out = io . BytesIO ( ) <EOL> byt = payload [ <NUM_LIT:16> : ] <EOL> length = len ( byt ) <EOL> cursor = <NUM_LIT:0> <EOL> while cursor < length : <EOL> block_size = struct . unpack_from ( '<STR_LIT>' , byt [ cursor : ] ) [ <NUM_LIT:0> ] <EOL> cursor += <NUM_LIT:4> <EOL> end = cursor + block_size <EOL> out . write ( snappy . decompress ( byt [ cursor : end ] ) ) <EOL> cursor = end <EOL> out . seek ( <NUM_LIT:0> ) <EOL> return out . read ( ) <EOL> else : <EOL> return snappy . decompress ( payload ) <EOL> def lz4_encode ( payload ) : <EOL> data = lz4f . compressFrame ( payload ) <EOL> header_size = <NUM_LIT:7> <EOL> if isinstance ( data [ <NUM_LIT:4> ] , int ) : <EOL> flg = data [ <NUM_LIT:4> ] <EOL> else : <EOL> flg = ord ( data [ <NUM_LIT:4> ] ) <EOL> content_size_bit = ( ( flg >> <NUM_LIT:3> ) & <NUM_LIT:1> ) <EOL> if content_size_bit : <EOL> header_size += <NUM_LIT:8> <EOL> hc = xxhash . xxh32 ( data [ <NUM_LIT:0> : header_size - <NUM_LIT:1> ] ) . digest ( ) [ - <NUM_LIT:2> : - <NUM_LIT:1> ] <EOL> return b'<STR_LIT>' . join ( [ <EOL> data [ <NUM_LIT:0> : header_size - <NUM_LIT:1> ] , <EOL> hc , <EOL> data [ header_size : ] <EOL> ] ) <EOL> def lz4_decode ( payload ) : <EOL> header_size = <NUM_LIT:7> <EOL> if isinstance ( payload [ <NUM_LIT:4> ] , int ) : <EOL> flg = payload [ <NUM_LIT:4> ] <EOL> else : <EOL> flg = ord ( payload [ <NUM_LIT:4> ] ) <EOL> content_size_bit = ( ( flg >> <NUM_LIT:3> ) & <NUM_LIT:1> ) <EOL> if content_size_bit : <EOL> header_size += <NUM_LIT:8> <EOL> hc = xxhash . xxh32 ( payload [ <NUM_LIT:4> : header_size - <NUM_LIT:1> ] ) . digest ( ) [ - <NUM_LIT:2> : - <NUM_LIT:1> ] <EOL> munged_payload = b'<STR_LIT>' . join ( [ <EOL> payload [ <NUM_LIT:0> : header_size - <NUM_LIT:1> ] , <EOL> hc , <EOL> payload [ header_size : ] <EOL> ] ) <EOL> cCtx = lz4f . createCompContext ( ) <EOL> data = lz4f . decompressFrame ( munged_payload , cCtx ) <EOL> return data [ '<STR_LIT>' ] </s>
