<s> '''<STR_LIT>''' <EOL> import re <EOL> from argparse import ArgumentParser <EOL> from cgi import FieldStorage <EOL> try : <EOL> from json import dumps <EOL> except ImportError : <EOL> from sys import path as sys_path <EOL> from os . path import join as path_join <EOL> from os . path import dirname <EOL> sys_path . append ( path_join ( dirname ( __file__ ) , '<STR_LIT>' ) ) <EOL> from ujson import dumps <EOL> from sys import stderr <EOL> from urlparse import urlparse <EOL> try : <EOL> from urlparse import parse_qs <EOL> except ImportError : <EOL> from cgi import parse_qs <EOL> from BaseHTTPServer import HTTPServer , BaseHTTPRequestHandler <EOL> ARGPARSER = ArgumentParser ( description = '<STR_LIT>' ) <EOL> ARGPARSER . add_argument ( '<STR_LIT>' , '<STR_LIT>' , type = int , default = <NUM_LIT> , <EOL> help = '<STR_LIT>' ) <EOL> def _tokens ( text ) : <EOL> anns = { } <EOL> if not text : <EOL> return anns <EOL> offset , aseq = <NUM_LIT:0> , <NUM_LIT:1> <EOL> for token in re . split ( '<STR_LIT>' , text ) : <EOL> if token and not token . isspace ( ) : <EOL> anns [ '<STR_LIT>' % aseq ] = { <EOL> '<STR_LIT:type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : ( ( offset , offset + len ( token ) ) , ) , <EOL> '<STR_LIT>' : ( token , ) , <EOL> } <EOL> aseq += <NUM_LIT:1> <EOL> offset += len ( token ) <EOL> return anns <EOL> class TokenizerHandler ( BaseHTTPRequestHandler ) : <EOL> def do_POST ( self ) : <EOL> field_storage = FieldStorage ( <EOL> headers = self . headers , <EOL> environ = { <EOL> '<STR_LIT>' : '<STR_LIT:POST>' , <EOL> '<STR_LIT>' : self . headers [ '<STR_LIT>' ] , <EOL> } , <EOL> fp = self . rfile ) <EOL> try : <EOL> json_dic = _tokens ( field_storage . value . decode ( '<STR_LIT:utf-8>' ) ) <EOL> except KeyError : <EOL> json_dic = { } <EOL> self . send_response ( <NUM_LIT:200> ) <EOL> self . send_header ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . end_headers ( ) <EOL> self . wfile . write ( dumps ( json_dic ) ) <EOL> print >> stderr , ( '<STR_LIT>' % len ( json_dic ) ) <EOL> def log_message ( self , format , * args ) : <EOL> return <EOL> def main ( args ) : <EOL> argp = ARGPARSER . parse_args ( args [ <NUM_LIT:1> : ] ) <EOL> server_class = HTTPServer <EOL> httpd = server_class ( ( '<STR_LIT:localhost>' , argp . port ) , TokenizerHandler ) <EOL> print >> stderr , '<STR_LIT>' % ( argp . port ) <EOL> try : <EOL> httpd . serve_forever ( ) <EOL> except KeyboardInterrupt : <EOL> pass <EOL> httpd . server_close ( ) <EOL> print >> stderr , '<STR_LIT>' <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> from sys import argv <EOL> exit ( main ( argv ) ) </s>
