<s> __author__ = '<STR_LIT>' <EOL> from treelib import tree <EOL> import fnmatch <EOL> import os <EOL> import zlib <EOL> import argparse <EOL> DEBUG = <NUM_LIT:0> <EOL> FILECOUNT = <NUM_LIT:0> <EOL> DIRCOUNT = <NUM_LIT:0> <EOL> DIR_ERRORLIST = [ ] <EOL> FILE_ERRORLIST = [ ] <EOL> PROFILING = <NUM_LIT:0> <EOL> if PROFILING == <NUM_LIT:1> : <EOL> import timeit <EOL> if PROFILING == <NUM_LIT:2> : <EOL> import cProfile <EOL> parser = argparse . ArgumentParser ( description = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , type = str , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , type = str , help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( ) <EOL> rootPath = args . abspath <EOL> pattern = args . pattern <EOL> folder_blacklist = [ ] <EOL> dir_tree = tree . Tree ( ) <EOL> dir_tree . create_node ( '<STR_LIT>' , rootPath ) <EOL> def crc32 ( data ) : <EOL> data = bytes ( data , '<STR_LIT>' ) <EOL> if DEBUG : <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' + str ( data ) ) <EOL> print ( '<STR_LIT>' + hex ( zlib . crc32 ( data ) & <NUM_LIT> ) ) <EOL> print ( '<STR_LIT>' ) <EOL> return hex ( zlib . crc32 ( data ) & <NUM_LIT> ) <EOL> parent = rootPath <EOL> i = <NUM_LIT:1> <EOL> start_depth = rootPath . count ( '<STR_LIT:/>' ) <EOL> def get_noteid ( depth , root , dir ) : <EOL> """<STR_LIT>""" <EOL> return str ( str ( depth ) + '<STR_LIT:_>' + dir ) . replace ( "<STR_LIT:U+0020>" , "<STR_LIT:_>" ) + '<STR_LIT>' + crc32 ( os . path . join ( root , dir ) ) <EOL> def get_parentid ( current_depth , root , dir ) : <EOL> if current_depth == <NUM_LIT:0> : <EOL> return root <EOL> search_string = os . path . join ( root , dir ) <EOL> pos2 = search_string . rfind ( '<STR_LIT:/>' ) <EOL> pos1 = search_string . rfind ( '<STR_LIT:/>' , <NUM_LIT:0> , pos2 ) <EOL> parent_dir = search_string [ pos1 + <NUM_LIT:1> : pos2 ] <EOL> parentid = str ( current_depth - <NUM_LIT:1> ) + '<STR_LIT:_>' + parent_dir . replace ( "<STR_LIT:U+0020>" , "<STR_LIT:_>" ) + '<STR_LIT>' + crc32 ( root ) <EOL> return parentid <EOL> def print_node ( dir , node_id , parent_id ) : <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' + dir ) <EOL> print ( '<STR_LIT>' + node_id ) <EOL> print ( '<STR_LIT>' + parent_id ) <EOL> def crawler ( ) : <EOL> global DIRCOUNT <EOL> global FILECOUNT <EOL> for root , dirs , files in os . walk ( rootPath ) : <EOL> for dir in dirs : <EOL> current_depth = os . path . join ( root , dir ) . count ( '<STR_LIT:/>' ) - start_depth <EOL> if DEBUG : <EOL> print ( '<STR_LIT>' + os . path . join ( root , dir ) ) <EOL> node_id = get_noteid ( current_depth , root , dir ) <EOL> parent_id = str ( get_parentid ( current_depth , root , dir ) ) <EOL> if parent_id == str ( None ) : <EOL> DIR_ERRORLIST . append ( os . path . join ( root , dir ) ) <EOL> if DEBUG : <EOL> print_node ( dir , node_id , parent_id ) <EOL> dir_tree . create_node ( dir , node_id , parent_id ) <EOL> DIRCOUNT += <NUM_LIT:1> <EOL> for filename in fnmatch . filter ( files , pattern ) : <EOL> if dir in folder_blacklist : <EOL> continue <EOL> current_depth = os . path . join ( root , filename ) . count ( '<STR_LIT:/>' ) - start_depth <EOL> if DEBUG : <EOL> print ( '<STR_LIT>' + os . path . join ( root , filename ) ) <EOL> node_id = get_noteid ( current_depth , root , filename ) <EOL> parent_id = str ( get_parentid ( current_depth , root , filename ) ) <EOL> if parent_id == str ( None ) : <EOL> FILE_ERRORLIST . append ( os . path . join ( root , dir ) ) <EOL> if DEBUG : <EOL> print_node ( filename , node_id , parent_id ) <EOL> dir_tree . create_node ( filename , node_id , parent_id ) <EOL> FILECOUNT += <NUM_LIT:1> <EOL> if PROFILING == <NUM_LIT:0> : <EOL> crawler ( ) <EOL> if PROFILING == <NUM_LIT:1> : <EOL> t1 = timeit . Timer ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> print ( '<STR_LIT>' + str ( t1 . timeit ( number = <NUM_LIT:1> ) ) ) <EOL> if PROFILING == <NUM_LIT:2> : <EOL> cProfile . run ( "<STR_LIT>" ) <EOL> print ( '<STR_LIT>' + str ( FILECOUNT ) ) <EOL> print ( '<STR_LIT>' + str ( DIRCOUNT ) ) <EOL> if DIR_ERRORLIST : <EOL> for item in DIR_ERRORLIST : <EOL> print ( item ) <EOL> else : <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> if FILE_ERRORLIST : <EOL> for item in FILE_ERRORLIST : <EOL> print ( item ) <EOL> else : <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' + str ( len ( dir_tree . nodes ) ) ) <EOL> dir_tree . show ( ) </s>
