<s> from __future__ import with_statement <EOL> import sys <EOL> import os <EOL> import re <EOL> import codecs <EOL> try : <EOL> import xml . etree . ElementTree as ET <EOL> except ImportError : <EOL> import cElementTree as ET <EOL> INSERTED_ELEMENT_TAG = "<STR_LIT>" <EOL> INPUT_ENCODING = "<STR_LIT>" <EOL> OUTPUT_ENCODING = "<STR_LIT>" <EOL> options = None <EOL> newline_wrap_element = set ( [ <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT:P>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT:title>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT:p>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ] ) <EOL> space_wrap_element = set ( [ <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT:email>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT:label>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ] ) <EOL> strip_element = newline_wrap_element | space_wrap_element <EOL> class Standoff : <EOL> def __init__ ( self , element , start , end ) : <EOL> self . element = element <EOL> self . start = start <EOL> self . end = end <EOL> def txt ( s ) : <EOL> return s if s is not None else "<STR_LIT>" <EOL> def text_and_standoffs ( e ) : <EOL> strings , standoffs = [ ] , [ ] <EOL> _text_and_standoffs ( e , <NUM_LIT:0> , strings , standoffs ) <EOL> text = "<STR_LIT>" . join ( strings ) <EOL> return text , standoffs <EOL> def _text_and_standoffs ( e , curroff , strings , standoffs ) : <EOL> startoff = curroff <EOL> so = Standoff ( e , <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> standoffs . append ( so ) <EOL> if e . text is not None and e . text != "<STR_LIT>" : <EOL> strings . append ( e . text ) <EOL> curroff += len ( e . text ) <EOL> curroff = _subelem_text_and_standoffs ( e , curroff , strings , standoffs ) <EOL> so . start = startoff <EOL> so . end = curroff <EOL> return curroff <EOL> def _subelem_text_and_standoffs ( e , curroff , strings , standoffs ) : <EOL> startoff = curroff <EOL> for s in e : <EOL> curroff = _text_and_standoffs ( s , curroff , strings , standoffs ) <EOL> if s . tail is not None and s . tail != "<STR_LIT>" : <EOL> strings . append ( s . tail ) <EOL> curroff += len ( s . tail ) <EOL> return curroff <EOL> def preceding_space ( pos , text , rewritten = { } ) : <EOL> while pos > <NUM_LIT:0> : <EOL> pos -= <NUM_LIT:1> <EOL> if pos not in rewritten : <EOL> return text [ pos ] . isspace ( ) <EOL> elif rewritten [ pos ] is not None : <EOL> return rewritten [ pos ] . isspace ( ) <EOL> else : <EOL> pass <EOL> return True <EOL> def following_space ( pos , text , rewritten = { } ) : <EOL> while pos < len ( text ) : <EOL> if pos not in rewritten : <EOL> return text [ pos ] . isspace ( ) <EOL> elif rewritten [ pos ] is not None : <EOL> return rewritten [ pos ] . isspace ( ) <EOL> else : <EOL> pass <EOL> pos += <NUM_LIT:1> <EOL> return True <EOL> def preceding_linebreak ( pos , text , rewritten = { } ) : <EOL> if pos >= len ( text ) : <EOL> return True <EOL> while pos > <NUM_LIT:0> : <EOL> pos -= <NUM_LIT:1> <EOL> c = rewritten . get ( pos , text [ pos ] ) <EOL> if c == "<STR_LIT:\n>" : <EOL> return True <EOL> elif c is not None and not c . isspace ( ) : <EOL> return False <EOL> else : <EOL> pass <EOL> return True <EOL> def following_linebreak ( pos , text , rewritten = { } ) : <EOL> while pos < len ( text ) : <EOL> c = rewritten . get ( pos , text [ pos ] ) <EOL> if c == "<STR_LIT:\n>" : <EOL> return True <EOL> elif c is not None and not c . isspace ( ) : <EOL> return False <EOL> else : <EOL> pass <EOL> pos += <NUM_LIT:1> <EOL> return True <EOL> def index_in_parent ( e , p ) : <EOL> """<STR_LIT>""" <EOL> index = None <EOL> for i in range ( len ( p ) ) : <EOL> if p [ i ] == e : <EOL> index = i <EOL> break <EOL> assert i is not None , "<STR_LIT>" <EOL> return i <EOL> def space_normalize ( root , text = None , standoffs = None ) : <EOL> """<STR_LIT>""" <EOL> if text is None or standoffs is None : <EOL> text , standoffs = text_and_standoffs ( root ) <EOL> for so in standoffs : <EOL> e = so . element <EOL> if e . text is not None and e . text != "<STR_LIT>" : <EOL> e . text = re . sub ( r'<STR_LIT>' , '<STR_LIT:U+0020>' , e . text ) <EOL> if e . tail is not None and e . tail != "<STR_LIT>" : <EOL> e . tail = re . sub ( r'<STR_LIT>' , '<STR_LIT:U+0020>' , e . tail ) <EOL> def strip_elements ( root , elements_to_strip = set ( ) , text = None , standoffs = None ) : <EOL> """<STR_LIT>""" <EOL> if text is None or standoffs is None : <EOL> text , standoffs = text_and_standoffs ( root ) <EOL> rewritten = { } <EOL> for so in standoffs : <EOL> e = so . element <EOL> if e . tag == INSERTED_ELEMENT_TAG : <EOL> continue <EOL> if ( ( e . text is not None and e . text != "<STR_LIT>" and e . text [ <NUM_LIT:0> ] . isspace ( ) ) and <EOL> ( element_in_set ( e , elements_to_strip ) or <EOL> preceding_space ( so . start , text , rewritten ) ) ) : <EOL> l = <NUM_LIT:0> <EOL> while l < len ( e . text ) and e . text [ l ] . isspace ( ) : <EOL> l += <NUM_LIT:1> <EOL> space , end = e . text [ : l ] , e . text [ l : ] <EOL> for i in range ( l ) : <EOL> assert so . start + i not in rewritten , "<STR_LIT>" % ( so . start + i ) <EOL> rewritten [ so . start + i ] = None <EOL> e . text = end <EOL> if len ( e ) == <NUM_LIT:0> : <EOL> if ( ( e . text is not None and e . text != "<STR_LIT>" and e . text [ - <NUM_LIT:1> ] . isspace ( ) ) and <EOL> ( element_in_set ( e , elements_to_strip ) or <EOL> following_space ( so . end , text , rewritten ) ) ) : <EOL> l = <NUM_LIT:0> <EOL> while l < len ( e . text ) and e . text [ - l - <NUM_LIT:1> ] . isspace ( ) : <EOL> l += <NUM_LIT:1> <EOL> start , space = e . text [ : - l ] , e . text [ - l : ] <EOL> for i in range ( l ) : <EOL> o = so . end - i - <NUM_LIT:1> <EOL> assert o not in rewritten , "<STR_LIT>" <EOL> rewritten [ o ] = None <EOL> e . text = start <EOL> else : <EOL> c = e [ - <NUM_LIT:1> ] <EOL> if ( ( c . tail is not None and c . tail != "<STR_LIT>" and c . tail [ - <NUM_LIT:1> ] . isspace ( ) ) and <EOL> ( element_in_set ( e , elements_to_strip ) or <EOL> following_space ( so . end , text , rewritten ) ) ) : <EOL> l = <NUM_LIT:0> <EOL> while l < len ( c . tail ) and c . tail [ - l - <NUM_LIT:1> ] . isspace ( ) : <EOL> l += <NUM_LIT:1> <EOL> start , space = c . tail [ : - l ] , c . tail [ - l : ] <EOL> for i in range ( l ) : <EOL> o = so . end - i - <NUM_LIT:1> <EOL> assert o not in rewritten , "<STR_LIT>" <EOL> rewritten [ o ] = None <EOL> c . tail = start <EOL> def trim_tails ( root ) : <EOL> """<STR_LIT>""" <EOL> text , standoffs = text_and_standoffs ( root ) <EOL> for so in standoffs : <EOL> e = so . element <EOL> if ( e . tail is not None and e . tail != "<STR_LIT>" and e . tail [ <NUM_LIT:0> ] . isspace ( ) and <EOL> preceding_space ( so . end , text ) ) : <EOL> l = <NUM_LIT:0> <EOL> while l < len ( e . tail ) and e . tail [ l ] . isspace ( ) : <EOL> l += <NUM_LIT:1> <EOL> space , end = e . tail [ : l ] , e . tail [ l : ] <EOL> e . tail = end <EOL> def reduce_space ( root , elements_to_strip = set ( ) ) : <EOL> """<STR_LIT>""" <EOL> text , standoffs = text_and_standoffs ( root ) <EOL> strip_elements ( root , elements_to_strip , text , standoffs ) <EOL> trim_tails ( root ) <EOL> space_normalize ( root , text , standoffs ) <EOL> def element_in_set ( e , s ) : <EOL> if e . tag [ <NUM_LIT:0> ] == "<STR_LIT:{>" : <EOL> tag = re . sub ( r'<STR_LIT>' , '<STR_LIT>' , e . tag ) <EOL> else : <EOL> tag = e . tag <EOL> return tag in s <EOL> def process ( fn ) : <EOL> global strip_element <EOL> global options <EOL> if fn == "<STR_LIT:->" : <EOL> fn = "<STR_LIT>" <EOL> try : <EOL> tree = ET . parse ( fn ) <EOL> except : <EOL> print >> sys . stderr , "<STR_LIT>" % fn <EOL> raise <EOL> root = tree . getroot ( ) <EOL> reduce_space ( root , strip_element ) <EOL> text , standoffs = text_and_standoffs ( root ) <EOL> respace = { } <EOL> for so in standoffs : <EOL> e = so . element <EOL> if element_in_set ( e , newline_wrap_element ) : <EOL> if not ( so . start in respace and ( respace [ so . start ] [ <NUM_LIT:0> ] == "<STR_LIT:\n>" and <EOL> respace [ so . start ] [ <NUM_LIT:1> ] == False ) ) : <EOL> respace [ so . start ] = ( "<STR_LIT:\n>" , True ) <EOL> respace [ so . end ] = ( "<STR_LIT:\n>" , False ) <EOL> elif element_in_set ( e , space_wrap_element ) : <EOL> if not ( so . start in respace and ( respace [ so . start ] [ <NUM_LIT:0> ] == "<STR_LIT:\n>" or <EOL> respace [ so . start ] [ <NUM_LIT:1> ] == False ) ) : <EOL> respace [ so . start ] = ( "<STR_LIT:U+0020>" , True ) <EOL> if not ( so . end in respace and respace [ so . end ] [ <NUM_LIT:0> ] == "<STR_LIT:\n>" ) : <EOL> respace [ so . end ] = ( "<STR_LIT:U+0020>" , False ) <EOL> rewritten = { } <EOL> filtered = { } <EOL> for pos in sorted ( respace . keys ( ) ) : <EOL> if respace [ pos ] [ <NUM_LIT:0> ] == "<STR_LIT:U+0020>" : <EOL> if not ( preceding_space ( pos , text , rewritten ) or <EOL> following_space ( pos , text , rewritten ) ) : <EOL> filtered [ pos ] = respace [ pos ] <EOL> rewritten [ pos - <NUM_LIT:1> ] = "<STR_LIT:U+0020>" <EOL> else : <EOL> assert respace [ pos ] [ <NUM_LIT:0> ] == "<STR_LIT:\n>" , "<STR_LIT>" <EOL> if not ( preceding_linebreak ( pos , text , rewritten ) or <EOL> following_linebreak ( pos , text , rewritten ) ) : <EOL> filtered [ pos ] = respace [ pos ] <EOL> rewritten [ pos - <NUM_LIT:1> ] = "<STR_LIT:\n>" <EOL> respace = filtered <EOL> parent_map = { } <EOL> for parent in root . getiterator ( ) : <EOL> for child in parent : <EOL> parent_map [ child ] = parent <EOL> end_map = { } <EOL> for so in standoffs : <EOL> if so . end not in end_map : <EOL> end_map [ so . end ] = [ ] <EOL> end_map [ so . end ] . append ( so ) <EOL> for so in standoffs : <EOL> if so . start in respace and respace [ so . start ] [ <NUM_LIT:1> ] == True : <EOL> e = so . element <EOL> assert e in parent_map , "<STR_LIT>" <EOL> p = parent_map [ e ] <EOL> i = index_in_parent ( e , p ) <EOL> rse = ET . Element ( INSERTED_ELEMENT_TAG ) <EOL> rse . text = respace [ so . start ] [ <NUM_LIT:0> ] <EOL> p . insert ( i , rse ) <EOL> del respace [ so . start ] <EOL> if so . end in respace and respace [ so . end ] [ <NUM_LIT:1> ] == False : <EOL> maxlen = max ( [ s . end - s . start for s in end_map [ so . end ] ] ) <EOL> if so . end - so . start != maxlen : <EOL> continue <EOL> longest = [ s for s in end_map [ so . end ] if s . end - s . start == maxlen ] <EOL> if so != longest [ <NUM_LIT:0> ] : <EOL> continue <EOL> e = so . element <EOL> assert e in parent_map , "<STR_LIT>" <EOL> p = parent_map [ e ] <EOL> i = index_in_parent ( e , p ) <EOL> rse = ET . Element ( INSERTED_ELEMENT_TAG ) <EOL> rse . text = respace [ so . end ] [ <NUM_LIT:0> ] <EOL> p . insert ( i + <NUM_LIT:1> , rse ) <EOL> rse . tail = e . tail <EOL> e . tail = "<STR_LIT>" <EOL> del respace [ so . end ] <EOL> assert len ( respace ) == <NUM_LIT:0> , "<STR_LIT>" % str ( respace ) <EOL> strip_elements ( root ) <EOL> trim_tails ( root ) <EOL> if options . stdout : <EOL> tree . write ( sys . stdout , encoding = OUTPUT_ENCODING ) <EOL> return True <EOL> if options is not None and options . directory is not None : <EOL> output_dir = options . directory <EOL> else : <EOL> output_dir = "<STR_LIT>" <EOL> output_fn = os . path . join ( output_dir , os . path . basename ( fn ) ) <EOL> if output_fn == fn and not options . overwrite : <EOL> print >> sys . stderr , '<STR_LIT>' % fn <EOL> else : <EOL> try : <EOL> with open ( output_fn , '<STR_LIT:w>' ) as of : <EOL> tree . write ( of , encoding = OUTPUT_ENCODING ) <EOL> except IOError , ex : <EOL> print >> sys . stderr , '<STR_LIT>' % ex <EOL> return True <EOL> def argparser ( ) : <EOL> import argparse <EOL> ap = argparse . ArgumentParser ( description = '<STR_LIT>' ) <EOL> ap . add_argument ( '<STR_LIT>' , '<STR_LIT>' , default = None , metavar = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> ap . add_argument ( '<STR_LIT>' , '<STR_LIT>' , default = False , action = '<STR_LIT:store_true>' , help = '<STR_LIT>' ) <EOL> ap . add_argument ( '<STR_LIT>' , '<STR_LIT>' , default = False , action = '<STR_LIT:store_true>' , help = '<STR_LIT>' ) <EOL> ap . add_argument ( '<STR_LIT:file>' , nargs = '<STR_LIT:+>' , help = '<STR_LIT>' ) <EOL> return ap <EOL> def main ( argv ) : <EOL> global options <EOL> options = argparser ( ) . parse_args ( argv [ <NUM_LIT:1> : ] ) <EOL> for fn in options . file : <EOL> process ( fn ) <EOL> return <NUM_LIT:0> <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> sys . exit ( main ( sys . argv ) ) </s>
