<s> from bottle import run , debug , route , abort , redirect , request , static_file , post , redirect <EOL> import shlex <EOL> import collections <EOL> import whoosh <EOL> from whoosh . fields import SchemaClass <EOL> from whoosh import index <EOL> from whoosh . qparser import QueryParser <EOL> import ConfigParser <EOL> import os <EOL> import argparse <EOL> import jinja2 <EOL> import subprocess <EOL> import shlex <EOL> import time <EOL> import calendar <EOL> import re <EOL> import pprint <EOL> import itertools <EOL> import datetime <EOL> import hashlib <EOL> os . environ [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> DIR_CONFIG_FILE_NAME = "<STR_LIT>" <EOL> FILE_READ_BLOCK_SIZE = <NUM_LIT:2> ** <NUM_LIT:20> <EOL> config = ConfigParser . ConfigParser ( ) <EOL> def datetime_format ( value , format = "<STR_LIT>" ) : <EOL> return time . strftime ( format , time . localtime ( value ) ) <EOL> jinja_env = jinja2 . Environment ( loader = jinja2 . FileSystemLoader ( '<STR_LIT>' ) , <EOL> trim_blocks = True ) <EOL> jinja_env . filters [ '<STR_LIT>' ] = datetime_format <EOL> markup_file_suffixes = set ( ) <EOL> markupserve_index = None <EOL> class MarkupServeSchema ( SchemaClass ) : <EOL> path = whoosh . fields . ID ( stored = True ) <EOL> title = whoosh . fields . TEXT ( stored = True ) <EOL> content = whoosh . fields . TEXT ( <EOL> analyzer = ( whoosh . analysis . StemmingAnalyzer ( ) ) , <EOL> stored = True ) <EOL> file_hash = whoosh . fields . ID ( stored = True ) <EOL> def safe_unicode ( obj , * args ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return unicode ( obj , * args ) <EOL> except UnicodeDecodeError : <EOL> ascii_text = str ( obj ) . encode ( '<STR_LIT>' ) <EOL> return unicode ( ascii_text ) <EOL> def find_program ( program ) : <EOL> for path in os . environ [ "<STR_LIT>" ] . split ( os . pathsep ) : <EOL> executable = os . path . join ( path , program ) <EOL> if os . path . exists ( executable ) and os . access ( executable , os . X_OK ) : <EOL> return os . path . abspath ( executable ) <EOL> return None <EOL> def last_modified_string ( file_path ) : <EOL> return time . strftime ( "<STR_LIT>" , time . localtime ( <EOL> os . path . getmtime ( file_path ) ) ) <EOL> def file_path_to_server_path ( path , root ) : <EOL> if path is None : <EOL> return '<STR_LIT>' <EOL> return os . path . join ( "<STR_LIT>" , os . path . relpath ( path , root ) ) <EOL> def view_calendar ( path , parent_path , root , config ) : <EOL> files = os . listdir ( path ) <EOL> try : <EOL> file_prefix = config . get ( "<STR_LIT>" , "<STR_LIT>" ) . strip ( '<STR_LIT:">' ) . strip ( "<STR_LIT:'>" ) <EOL> except ConfigParser . Error : <EOL> file_prefix = "<STR_LIT>" <EOL> try : <EOL> file_suffix = config . get ( "<STR_LIT>" , "<STR_LIT>" ) . strip ( '<STR_LIT:">' ) . strip ( "<STR_LIT:'>" ) <EOL> except ConfigParser . Error : <EOL> file_suffix = "<STR_LIT>" <EOL> filename_regex = re . compile ( "<STR_LIT>" % <EOL> ( file_prefix , file_suffix ) ) <EOL> def regex_match_function ( x ) : <EOL> match = filename_regex . match ( x ) <EOL> if match is not None : <EOL> return match . groups ( ) <EOL> else : <EOL> return None <EOL> file_dates = [ match for match in map ( regex_match_function , files ) ] <EOL> files_by_month = { } <EOL> for filename , date in itertools . izip ( files , file_dates ) : <EOL> if date is None : <EOL> continue <EOL> file_path = os . path . join ( path , filename ) <EOL> year , month , day = map ( int , date ) <EOL> date = datetime . date ( year , month , day ) <EOL> year_and_month = ( year , month ) <EOL> if year_and_month not in files_by_month : <EOL> files_by_month [ year_and_month ] = { } <EOL> files_by_month [ year_and_month ] [ date ] = file_path <EOL> calendars = { } <EOL> cal = calendar . Calendar ( ) <EOL> cal . setfirstweekday ( <NUM_LIT:6> ) <EOL> for year_and_month , dates in files_by_month . items ( ) : <EOL> year , month = year_and_month <EOL> if year not in calendars : <EOL> calendars [ year ] = { } <EOL> calendars [ year ] [ month ] = [ ] <EOL> for week in cal . monthdatescalendar ( year , month ) : <EOL> week_list = [ ] <EOL> for date in week : <EOL> date_info = { } <EOL> if date in files_by_month [ year_and_month ] : <EOL> date_info [ "<STR_LIT>" ] = file_path_to_server_path ( <EOL> files_by_month [ year_and_month ] [ date ] , root ) <EOL> date_info [ "<STR_LIT>" ] = date . day <EOL> if date . month == month : <EOL> date_info [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> else : <EOL> date_info [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> week_list . append ( date_info ) <EOL> calendars [ year ] [ month ] . append ( week_list ) <EOL> template = jinja_env . get_template ( "<STR_LIT>" ) <EOL> return template . render ( <EOL> calendars = calendars , <EOL> month_names = calendar . month_name , <EOL> path = path , <EOL> parent_path = file_path_to_server_path ( parent_path , root ) ) <EOL> def view_dir ( path , parent_path , root , sorted_by , reverse ) : <EOL> files = os . listdir ( path ) <EOL> listable_files = [ ] <EOL> for filename in files : <EOL> if filename [ <NUM_LIT:0> ] == '<STR_LIT:.>' or os . path . splitext ( filename ) [ <NUM_LIT:1> ] == "<STR_LIT>" : <EOL> continue <EOL> file_info = { } <EOL> file_path = os . path . join ( path , filename ) <EOL> file_info [ "<STR_LIT:name>" ] = filename <EOL> if os . path . isdir ( file_path ) : <EOL> file_info [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> else : <EOL> file_info [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> file_info [ "<STR_LIT>" ] = file_path_to_server_path ( file_path , root ) <EOL> file_info [ "<STR_LIT>" ] = last_modified_string ( file_path ) <EOL> listable_files . append ( file_info ) <EOL> if sorted_by is not None : <EOL> listable_files . sort ( key = lambda x : x [ sorted_by ] , reverse = reverse ) <EOL> if parent_path != None : <EOL> parent_path_info = { <EOL> "<STR_LIT:name>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : file_path_to_server_path ( parent_path , root ) , <EOL> "<STR_LIT>" : last_modified_string ( parent_path ) , <EOL> "<STR_LIT>" : "<STR_LIT>" <EOL> } <EOL> listable_files . insert ( <NUM_LIT:0> , parent_path_info ) <EOL> template = jinja_env . get_template ( "<STR_LIT>" ) <EOL> page_uri = file_path_to_server_path ( path , root ) <EOL> return template . render ( files = listable_files , path = path , <EOL> page_uri = page_uri , sorted_by = sorted_by , <EOL> reverse = reverse ) <EOL> def view_file ( path , root ) : <EOL> file_suffix = os . path . splitext ( path ) [ <NUM_LIT:1> ] <EOL> parent_dir = os . path . abspath ( os . path . join ( path , os . pardir ) ) <EOL> files_in_dir = filter ( <EOL> lambda x : os . path . splitext ( x ) [ <NUM_LIT:1> ] != "<STR_LIT>" and x [ <NUM_LIT:0> ] != '<STR_LIT:.>' , <EOL> os . listdir ( parent_dir ) ) <EOL> files_in_dir . sort ( ) <EOL> file_index = files_in_dir . index ( os . path . relpath ( path , parent_dir ) ) <EOL> def make_path_struct ( filename ) : <EOL> path_dict = { <EOL> "<STR_LIT:name>" : os . path . splitext ( filename ) [ <NUM_LIT:0> ] , <EOL> "<STR_LIT>" : '<STR_LIT:/>' . join ( ( "<STR_LIT>" , os . path . relpath ( parent_dir , root ) , <EOL> filename ) ) <EOL> } <EOL> return path_dict <EOL> if file_index > <NUM_LIT:0> : <EOL> prev_path = make_path_struct ( files_in_dir [ file_index - <NUM_LIT:1> ] ) <EOL> else : <EOL> prev_path = None <EOL> if file_index < len ( files_in_dir ) - <NUM_LIT:1> : <EOL> next_path = make_path_struct ( files_in_dir [ file_index + <NUM_LIT:1> ] ) <EOL> else : <EOL> next_path = None <EOL> if file_suffix not in markup_file_suffixes : <EOL> return static_file ( path , root = "<STR_LIT:/>" ) <EOL> converter_bin = os . path . abspath ( config . get ( <EOL> "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> command = shlex . split ( '<STR_LIT>' % ( converter_bin , path ) ) <EOL> render_process = subprocess . Popen ( command , stdout = subprocess . PIPE , <EOL> stderr = subprocess . PIPE ) <EOL> ( output , error ) = render_process . communicate ( ) <EOL> if render_process . returncode != <NUM_LIT:0> : <EOL> abort ( <NUM_LIT> , "<STR_LIT>" <EOL> % ( path , render_process . returncode , output , error ) ) <EOL> template = jinja_env . get_template ( "<STR_LIT>" ) <EOL> filename = os . path . splitext ( os . path . basename ( path ) ) [ <NUM_LIT:0> ] <EOL> parent_path = "<STR_LIT>" + os . path . relpath ( parent_dir , root ) <EOL> output = output . decode ( "<STR_LIT:utf-8>" ) <EOL> return template . render ( <EOL> filename = filename , last_modified = os . path . getmtime ( path ) , content = output , <EOL> parent = parent_path , prev = prev_path , next = next_path ) <EOL> @ route ( "<STR_LIT>" ) <EOL> def serve_static_file ( filename ) : <EOL> static_root = os . path . join ( os . path . dirname ( __file__ ) , "<STR_LIT>" ) <EOL> return static_file ( filename , root = static_root ) <EOL> def grep_search ( search_terms , document_root ) : <EOL> command = shlex . split ( '<STR_LIT>' % ( search_terms , document_root ) ) <EOL> grep_process = subprocess . Popen ( command , stdout = subprocess . PIPE , <EOL> stderr = subprocess . PIPE ) <EOL> ( output , error ) = grep_process . communicate ( ) <EOL> if grep_process . returncode not in [ <NUM_LIT:0> , <NUM_LIT:1> ] : <EOL> abort ( <NUM_LIT> , "<STR_LIT>" % ( <EOL> command , grep_process . returncode , output , error ) ) <EOL> results = collections . defaultdict ( list ) <EOL> for match in output . split ( '<STR_LIT:\n>' ) : <EOL> if len ( match ) == <NUM_LIT:0> : <EOL> continue <EOL> ( filename , delim , line_text ) = match . partition ( '<STR_LIT::>' ) <EOL> file_extension = os . path . splitext ( filename ) [ <NUM_LIT:1> ] <EOL> if file_extension not in markup_file_suffixes : <EOL> continue <EOL> filename = os . path . relpath ( filename , document_root ) <EOL> results [ filename ] . append ( unicode ( line_text , "<STR_LIT:utf8>" ) ) <EOL> return results <EOL> def index_search ( search_terms , document_root ) : <EOL> qp = QueryParser ( "<STR_LIT:content>" , schema = markupserve_index . schema ) <EOL> query = qp . parse ( safe_unicode ( search_terms ) ) <EOL> results = collections . defaultdict ( list ) <EOL> split_terms = shlex . split ( search_terms ) <EOL> with markupserve_index . searcher ( ) as searcher : <EOL> query_results = searcher . search ( query , limit = None ) <EOL> for result in query_results : <EOL> filename = result [ "<STR_LIT:path>" ] . decode ( "<STR_LIT:utf-8>" ) . decode ( "<STR_LIT>" ) <EOL> results [ filename ] . append ( <EOL> result . highlights ( "<STR_LIT:content>" ) . decode ( "<STR_LIT:utf-8>" ) <EOL> . decode ( "<STR_LIT>" ) ) <EOL> return results <EOL> @ route ( "<STR_LIT>" ) <EOL> def search ( ) : <EOL> search_terms = request . GET . dict [ "<STR_LIT>" ] [ <NUM_LIT:0> ] <EOL> document_root = os . path . expanduser ( config . get ( <EOL> "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> if markupserve_index is None : <EOL> search_function = grep_search <EOL> else : <EOL> search_function = index_search <EOL> results = search_function ( search_terms , document_root ) <EOL> template = jinja_env . get_template ( "<STR_LIT>" ) <EOL> return template . render ( terms = search_terms , results = results ) <EOL> @ route ( "<STR_LIT>" ) <EOL> def view ( path ) : <EOL> """<STR_LIT>""" <EOL> if "<STR_LIT>" in request . GET . dict : <EOL> sorted_by = request . GET . dict [ "<STR_LIT>" ] [ <NUM_LIT:0> ] <EOL> else : <EOL> sorted_by = None <EOL> reverse = "<STR_LIT>" in request . GET and ( <EOL> int ( request . GET . dict [ "<STR_LIT>" ] [ <NUM_LIT:0> ] ) == <NUM_LIT:1> ) <EOL> document_root = os . path . expanduser ( config . get ( <EOL> "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> absolute_path = os . path . abspath ( document_root + "<STR_LIT:/>" + path ) <EOL> if not os . path . exists ( absolute_path ) : <EOL> abort ( <NUM_LIT> , "<STR_LIT>" % ( absolute_path ) ) <EOL> if os . path . isdir ( absolute_path ) : <EOL> if path != "<STR_LIT:/>" : <EOL> parent_path = os . path . abspath ( absolute_path + "<STR_LIT:/>" + os . pardir ) <EOL> else : <EOL> parent_path = None <EOL> markupserve_dir_config_file = os . path . join ( absolute_path , <EOL> DIR_CONFIG_FILE_NAME ) <EOL> if os . path . exists ( markupserve_dir_config_file ) : <EOL> dir_config = ConfigParser . ConfigParser ( ) <EOL> parsed_files = dir_config . read ( [ markupserve_dir_config_file ] ) <EOL> if len ( parsed_files ) != <NUM_LIT:1> : <EOL> abort ( <NUM_LIT> , "<STR_LIT>" % <EOL> ( markupserve_dir_config_file ) ) <EOL> try : <EOL> dir_style = dir_config . get ( "<STR_LIT>" , "<STR_LIT:name>" ) <EOL> except ConfigParser . Error , e : <EOL> abort ( <NUM_LIT> , "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if dir_style == "<STR_LIT>" : <EOL> return view_calendar ( absolute_path , parent_path , document_root , <EOL> dir_config ) <EOL> return view_dir ( absolute_path , parent_path , document_root , <EOL> sorted_by , reverse ) <EOL> else : <EOL> return view_file ( absolute_path , document_root ) <EOL> @ route ( "<STR_LIT:/>" ) <EOL> @ route ( "<STR_LIT>" ) <EOL> def view_index ( ) : <EOL> return view ( "<STR_LIT:/>" ) <EOL> def hash_file_contents ( contents ) : <EOL> return hashlib . md5 ( contents ) . hexdigest ( ) <EOL> def add_file_to_index ( filename , document_root , writer ) : <EOL> file_basename = os . path . basename ( filename ) <EOL> filename_root = os . path . splitext ( file_basename ) [ <NUM_LIT:0> ] <EOL> with open ( filename , '<STR_LIT:r>' ) as fp : <EOL> file_contents = fp . read ( ) <EOL> file_hash = hash_file_contents ( file_contents ) <EOL> writer . add_document ( <EOL> title = safe_unicode ( filename_root ) , <EOL> content = safe_unicode ( file_contents ) , <EOL> file_hash = safe_unicode ( file_hash ) , <EOL> path = safe_unicode ( <EOL> os . path . relpath ( filename , document_root ) ) ) <EOL> def remove_file_from_index ( filename , document_root , writer ) : <EOL> writer . delete_by_term ( <EOL> '<STR_LIT:path>' , safe_unicode ( os . path . relpath ( filename , document_root ) ) ) <EOL> def markup_files_in_subtree ( root ) : <EOL> for dirpath , dirnames , filenames in os . walk ( root ) : <EOL> for filename in filenames : <EOL> extension = os . path . splitext ( filename ) [ <NUM_LIT:1> ] <EOL> if extension not in markup_file_suffixes : <EOL> continue <EOL> else : <EOL> yield os . path . join ( dirpath , filename ) <EOL> def build_index ( writer ) : <EOL> document_root = os . path . expanduser ( config . get ( <EOL> "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> for file_abspath in markup_files_in_subtree ( document_root ) : <EOL> add_file_to_index ( file_abspath , document_root , writer ) <EOL> @ post ( "<STR_LIT>" ) <EOL> def update_index ( ) : <EOL> document_root = os . path . expanduser ( config . get ( <EOL> "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> indexed_paths = set ( ) <EOL> to_index = set ( ) <EOL> print "<STR_LIT>" <EOL> if markupserve_index is None : <EOL> print "<STR_LIT>" <EOL> redirect ( '<STR_LIT:/>' ) <EOL> with markupserve_index . searcher ( ) as searcher : <EOL> writer = markupserve_index . writer ( ) <EOL> for fields in searcher . all_stored_fields ( ) : <EOL> indexed_path = os . path . join ( document_root , fields [ "<STR_LIT:path>" ] ) <EOL> indexed_paths . add ( indexed_path ) <EOL> if not os . path . exists ( indexed_path ) : <EOL> print "<STR_LIT>" % ( indexed_path ) <EOL> remove_file_from_index ( indexed_path , document_root , writer ) <EOL> else : <EOL> with open ( indexed_path , '<STR_LIT:r>' ) as fp : <EOL> file_hash = hash_file_contents ( fp . read ( ) ) <EOL> if file_hash != fields [ "<STR_LIT>" ] : <EOL> print "<STR_LIT>" % ( <EOL> indexed_path , file_hash , fields [ "<STR_LIT>" ] ) <EOL> remove_file_from_index ( indexed_path , document_root , writer ) <EOL> to_index . add ( indexed_path ) <EOL> for file_abspath in markup_files_in_subtree ( document_root ) : <EOL> if file_abspath in to_index : <EOL> print "<STR_LIT>" % ( file_abspath ) <EOL> add_file_to_index ( file_abspath , document_root , writer ) <EOL> elif file_abspath not in indexed_paths : <EOL> print "<STR_LIT>" % ( file_abspath ) <EOL> add_file_to_index ( file_abspath , document_root , writer ) <EOL> writer . commit ( ) <EOL> print "<STR_LIT>" <EOL> redirect ( '<STR_LIT:/>' ) <EOL> parser = argparse . ArgumentParser ( <EOL> description = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> parser . add_argument ( "<STR_LIT:-c>" , "<STR_LIT>" , default = "<STR_LIT>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> args = parser . parse_args ( ) <EOL> if not os . path . exists ( args . config ) : <EOL> exit ( "<STR_LIT>" % ( args . config ) ) <EOL> with open ( args . config , '<STR_LIT:r>' ) as fp : <EOL> config . readfp ( fp ) <EOL> if ( not config . has_section ( "<STR_LIT>" ) or <EOL> not config . has_option ( "<STR_LIT>" , "<STR_LIT>" ) or <EOL> not config . has_option ( "<STR_LIT>" , "<STR_LIT:port>" ) or <EOL> not config . has_option ( "<STR_LIT>" , "<STR_LIT>" ) or <EOL> not config . has_option ( "<STR_LIT>" , "<STR_LIT>" ) ) : <EOL> exit ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> converter_binary = config . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> if not os . path . exists ( converter_binary ) : <EOL> converter_binary = find_program ( converter_binary ) <EOL> if converter_binary != None : <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , converter_binary ) <EOL> else : <EOL> exit ( "<STR_LIT>" % ( converter_binary ) ) <EOL> for suffix in config . get ( "<STR_LIT>" , "<STR_LIT>" ) . split ( '<STR_LIT:U+002C>' ) : <EOL> markup_file_suffixes . add ( suffix . strip ( ) ) <EOL> port = config . getint ( "<STR_LIT>" , "<STR_LIT:port>" ) <EOL> if config . has_option ( "<STR_LIT>" , "<STR_LIT>" ) : <EOL> index_root = os . path . expanduser ( config . get ( "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> if os . path . isdir ( index_root ) : <EOL> print "<STR_LIT>" % ( index_root ) <EOL> markupserve_index = index . open_dir ( index_root ) <EOL> else : <EOL> print "<STR_LIT>" % ( index_root ) <EOL> os . makedirs ( index_root ) <EOL> markupserve_index = index . create_in ( <EOL> index_root , MarkupServeSchema ) <EOL> print "<STR_LIT>" <EOL> try : <EOL> writer = markupserve_index . writer ( ) <EOL> build_index ( writer ) <EOL> writer . commit ( ) <EOL> except whoosh . store . LockError , e : <EOL> print "<STR_LIT>" <EOL> debug ( True ) <EOL> run ( host = '<STR_LIT:localhost>' , port = port , server = "<STR_LIT>" ) </s>
