<s> import os <EOL> from time import time <EOL> import UserDict <EOL> import logging <EOL> import binascii <EOL> from cStringIO import StringIO <EOL> from itertools import chain <EOL> import weakref <EOL> import json <EOL> import functools <EOL> from easygit import EasyGit <EOL> from treetree import TreeTree <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> log . setLevel ( logging . DEBUG ) <EOL> class FeatureBlob ( object ) : <EOL> def __init__ ( self , blob ) : <EOL> self . blob = blob <EOL> def load ( self ) : <EOL> return json . loads ( self . blob . data ) <EOL> def save ( self , data ) : <EOL> self . blob . data = json . dumps ( data ) <EOL> nothing = object ( ) <EOL> def get ( self , key , default = nothing ) : <EOL> try : <EOL> return self . load ( ) [ key ] <EOL> except KeyError : <EOL> if default is not self . nothing : <EOL> return default <EOL> else : <EOL> raise <EOL> def __getitem__ ( self , key ) : <EOL> return self . get ( key ) <EOL> def __setitem__ ( self , key , value ) : <EOL> assert isinstance ( key , basestring ) <EOL> assert isinstance ( value , ( basestring , int ) ) <EOL> data = self . load ( ) <EOL> data [ key ] = value <EOL> self . save ( data ) <EOL> class GitStorage ( object ) : <EOL> commit_author = "<STR_LIT>" <EOL> @ classmethod <EOL> def create ( cls , repo_path ) : <EOL> if not os . path . isdir ( repo_path ) : <EOL> os . mkdir ( repo_path ) <EOL> eg = EasyGit . new_repo ( repo_path , bare = True ) <EOL> inodes = eg . root . new_tree ( '<STR_LIT>' ) <EOL> root_ls = eg . root . new_blob ( '<STR_LIT>' ) <EOL> root_sub = eg . root . new_tree ( '<STR_LIT>' ) <EOL> features_blob = eg . root . new_blob ( '<STR_LIT>' ) <EOL> features_blob . data = '<STR_LIT:{}>' <EOL> features = FeatureBlob ( features_blob ) <EOL> features [ '<STR_LIT>' ] = <NUM_LIT:1> <EOL> features [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> features [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> eg . commit ( cls . commit_author , '<STR_LIT>' ) <EOL> return cls ( repo_path ) <EOL> def __init__ ( self , repo_path , autocommit = True ) : <EOL> self . eg = EasyGit . open_repo ( repo_path ) <EOL> features = FeatureBlob ( self . eg . root [ '<STR_LIT>' ] ) <EOL> assert features . get ( '<STR_LIT>' , None ) == '<STR_LIT>' <EOL> assert features . get ( '<STR_LIT>' , None ) == '<STR_LIT>' <EOL> self . autocommit = autocommit <EOL> log . debug ( '<STR_LIT>' , <EOL> autocommit , self . eg . get_head_id ( ) ) <EOL> self . _inode_cache = { } <EOL> self . _inodes_tt = TreeTree ( self . eg . root [ '<STR_LIT>' ] , prefix = '<STR_LIT>' ) <EOL> def get_root ( self ) : <EOL> commit_tree = self . eg . root <EOL> root_ls = commit_tree [ '<STR_LIT>' ] <EOL> root_sub = commit_tree [ '<STR_LIT>' ] <EOL> root = StorageDir ( '<STR_LIT:root>' , root_ls , root_sub , '<STR_LIT:/>' , self , None ) <EOL> root . path = '<STR_LIT:/>' <EOL> return root <EOL> def get_inode ( self , name ) : <EOL> if name in self . _inode_cache : <EOL> inode = self . _inode_cache [ name ] ( ) <EOL> if inode is None : <EOL> del self . _inode_cache [ name ] <EOL> else : <EOL> return inode <EOL> inode_tree = self . _inodes_tt [ name [ <NUM_LIT:1> : ] ] <EOL> inode = StorageInode ( name , inode_tree , self ) <EOL> self . _inode_cache [ name ] = weakref . ref ( inode ) <EOL> return inode <EOL> def create_inode ( self ) : <EOL> features = FeatureBlob ( self . eg . root [ '<STR_LIT>' ] ) <EOL> next_inode_number = features [ '<STR_LIT>' ] <EOL> features [ '<STR_LIT>' ] = next_inode_number + <NUM_LIT:1> <EOL> inode_name = '<STR_LIT>' % next_inode_number <EOL> inode_tree = self . _inodes_tt . new_tree ( inode_name [ <NUM_LIT:1> : ] ) <EOL> inode_tree . new_blob ( '<STR_LIT>' ) . data = StorageInode . default_meta <EOL> return self . get_inode ( inode_name ) <EOL> def _remove_inode ( self , name ) : <EOL> if name in self . _inode_cache : <EOL> del self . _inode_cache [ name ] <EOL> def _autocommit ( self ) : <EOL> if self . autocommit : <EOL> self . commit ( "<STR_LIT>" ) <EOL> def commit ( self , message = None , amend = False , head_id = None , branch = '<STR_LIT>' ) : <EOL> log . info ( '<STR_LIT>' ) <EOL> if head_id is None : <EOL> head_id = self . eg . get_head_id ( branch ) <EOL> if amend : <EOL> git = self . eg . git <EOL> prev_commit = git . commit ( head_id ) <EOL> parents = prev_commit . parents <EOL> if message is None : <EOL> message = prev_commit . message <EOL> else : <EOL> parents = [ head_id ] <EOL> assert message is not None <EOL> self . eg . commit ( self . commit_author , message , parents , <EOL> branch = branch ) <EOL> class StorageDir ( object , UserDict . DictMixin ) : <EOL> is_dir = True <EOL> def __init__ ( self , name , ls_blob , sub_tree , path , storage , parent ) : <EOL> self . name = name <EOL> self . ls_blob = ls_blob <EOL> self . sub_tree = sub_tree <EOL> self . path = path <EOL> self . storage = storage <EOL> self . parent = parent <EOL> log . debug ( '<STR_LIT>' , name ) <EOL> def _iter_contents ( self ) : <EOL> return iter_entries ( self . ls_blob . data ) <EOL> def keys ( self ) : <EOL> for name , value in self . _iter_contents ( ) : <EOL> yield name <EOL> def __getitem__ ( self , key ) : <EOL> for name , value in self . _iter_contents ( ) : <EOL> if key == name : <EOL> break <EOL> else : <EOL> raise KeyError ( '<STR_LIT>' % repr ( key ) ) <EOL> if value == '<STR_LIT:/>' : <EOL> qname = quote ( name ) <EOL> child_ls = self . sub_tree [ qname + '<STR_LIT>' ] <EOL> try : <EOL> child_sub = self . sub_tree [ qname + '<STR_LIT>' ] <EOL> except KeyError : <EOL> child_sub = self . sub_tree . new_tree ( qname + '<STR_LIT>' ) <EOL> self . storage . _autocommit ( ) <EOL> return StorageDir ( name , child_ls , child_sub , <EOL> self . path + name + '<STR_LIT:/>' , <EOL> self . storage , self ) <EOL> else : <EOL> inode = self . storage . get_inode ( value ) <EOL> return StorageFile ( name , inode , self ) <EOL> def create_file ( self , name , inode = None ) : <EOL> check_filename ( name ) <EOL> if inode is None : <EOL> log . info ( '<STR_LIT>' , name , self . path ) <EOL> inode = self . storage . create_inode ( ) <EOL> else : <EOL> assert ( inode . storage is self . storage ) <EOL> log . info ( '<STR_LIT>' , <EOL> name , self . path , inode . name ) <EOL> inode [ '<STR_LIT>' ] += <NUM_LIT:1> <EOL> with self . ls_blob as b : <EOL> b . data += "<STR_LIT>" % ( quote ( name ) , inode . name ) <EOL> self . storage . _autocommit ( ) <EOL> return self [ name ] <EOL> def link_file ( self , name , src_file ) : <EOL> """<STR_LIT>""" <EOL> assert not src_file . is_dir <EOL> return self . create_file ( name , src_file . inode ) <EOL> def create_directory ( self , name ) : <EOL> check_filename ( name ) <EOL> log . info ( '<STR_LIT>' , repr ( name ) , repr ( self . path ) ) <EOL> qname = quote ( name ) <EOL> with self . sub_tree as st : <EOL> child_ls_blob = st . new_blob ( qname + '<STR_LIT>' ) <EOL> with self . ls_blob as b : <EOL> b . data += "<STR_LIT>" % qname <EOL> self . storage . _autocommit ( ) <EOL> return self [ name ] <EOL> def remove_ls_entry ( self , rm_name ) : <EOL> ls_data = '<STR_LIT>' <EOL> removed_count = <NUM_LIT:0> <EOL> for name , value in self . _iter_contents ( ) : <EOL> if name == rm_name : <EOL> log . debug ( '<STR_LIT>' , <EOL> repr ( rm_name ) , repr ( self . path ) ) <EOL> removed_count += <NUM_LIT:1> <EOL> else : <EOL> ls_data += '<STR_LIT>' % ( quote ( name ) , value ) <EOL> assert removed_count == <NUM_LIT:1> <EOL> with self . ls_blob as b : <EOL> b . data = ls_data <EOL> self . storage . _autocommit ( ) <EOL> def unlink ( self ) : <EOL> log . info ( '<STR_LIT>' , repr ( self . path ) ) <EOL> self . ls_blob . remove ( ) <EOL> self . sub_tree . remove ( ) <EOL> self . parent . remove_ls_entry ( self . name ) <EOL> self . storage . _autocommit ( ) <EOL> class StorageInode ( object ) : <EOL> blocksize = <NUM_LIT:64> * <NUM_LIT> <EOL> default_meta = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> int_meta = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:size>' ) <EOL> oct_meta = ( '<STR_LIT>' , ) <EOL> def __init__ ( self , name , tree , storage ) : <EOL> self . name = name <EOL> self . tree = tree <EOL> self . storage = storage <EOL> self . tt = TreeTree ( tree , prefix = '<STR_LIT>' ) <EOL> log . debug ( '<STR_LIT>' , name ) <EOL> def _read_meta ( self ) : <EOL> try : <EOL> meta_blob = self . tree [ '<STR_LIT>' ] <EOL> except KeyError : <EOL> meta_raw = self . default_meta <EOL> else : <EOL> meta_raw = meta_blob . data <EOL> return dict ( line . split ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> for line in meta_raw . strip ( ) . split ( '<STR_LIT:\n>' ) ) <EOL> def _write_meta ( self , meta_data ) : <EOL> meta_raw = '<STR_LIT>' . join ( '<STR_LIT>' % ( key , value ) <EOL> for key , value in sorted ( meta_data . items ( ) ) ) <EOL> self . tree . new_blob ( '<STR_LIT>' ) . data = meta_raw <EOL> self . storage . _autocommit ( ) <EOL> def __getitem__ ( self , key ) : <EOL> value = self . _read_meta ( ) [ key ] <EOL> if key in self . oct_meta : <EOL> value = int ( value , base = <NUM_LIT:8> ) <EOL> elif key in self . int_meta : <EOL> value = int ( value ) <EOL> return value <EOL> def __setitem__ ( self , key , value ) : <EOL> if key in self . oct_meta : <EOL> value = '<STR_LIT>' % value <EOL> elif key in self . int_meta : <EOL> value = '<STR_LIT>' % value <EOL> else : <EOL> raise NotImplementedError <EOL> meta_data = self . _read_meta ( ) <EOL> meta_data [ key ] = value <EOL> self . _write_meta ( meta_data ) <EOL> def read_block ( self , n ) : <EOL> block_name = str ( n ) <EOL> log . debug ( '<STR_LIT>' , block_name , self . name ) <EOL> try : <EOL> block = self . tt [ block_name ] <EOL> except KeyError : <EOL> return '<STR_LIT>' <EOL> else : <EOL> return block . data <EOL> def write_block ( self , n , data ) : <EOL> block_name = str ( n ) <EOL> log . debug ( '<STR_LIT>' , block_name , self . name ) <EOL> try : <EOL> block = self . tt [ block_name ] <EOL> except KeyError : <EOL> block = self . tt . new_blob ( block_name ) <EOL> block . data = data <EOL> self . storage . _autocommit ( ) <EOL> def delete_block ( self , n ) : <EOL> block_name = str ( n ) <EOL> log . debug ( '<STR_LIT>' , block_name , self . name ) <EOL> del self . tt [ block_name ] <EOL> self . storage . _autocommit ( ) <EOL> def read_data ( self , offset , length ) : <EOL> end = offset + length <EOL> eof = self [ '<STR_LIT:size>' ] <EOL> if end > eof : <EOL> end = eof <EOL> length = end - offset <EOL> if length <= <NUM_LIT:0> : <EOL> return '<STR_LIT>' <EOL> first_block = offset / self . blocksize <EOL> last_block = end / self . blocksize <EOL> output = StringIO ( ) <EOL> for n_block in range ( first_block , last_block + <NUM_LIT:1> ) : <EOL> block_offset = n_block * self . blocksize <EOL> fragment_offset = <NUM_LIT:0> <EOL> if n_block == first_block : <EOL> fragment_offset = offset - block_offset <EOL> fragment_end = self . blocksize <EOL> if n_block == last_block : <EOL> fragment_end = end - block_offset <EOL> block_data = self . read_block ( n_block ) <EOL> fragment = block_data [ fragment_offset : fragment_end ] <EOL> assert len ( fragment ) == fragment_end - fragment_offset <EOL> output . write ( fragment ) <EOL> output = output . getvalue ( ) <EOL> assert len ( output ) == length <EOL> return output <EOL> def write_data ( self , data , offset ) : <EOL> current_size = self [ '<STR_LIT:size>' ] <EOL> if current_size < offset : <EOL> self . truncate ( offset ) <EOL> log . info ( '<STR_LIT>' , <EOL> repr ( self . name ) , len ( data ) , offset ) <EOL> end = offset + len ( data ) <EOL> first_block = offset / self . blocksize <EOL> last_block = end / self . blocksize <EOL> for n_block in range ( first_block , last_block + <NUM_LIT:1> ) : <EOL> block_offset = n_block * self . blocksize <EOL> insert_offset = <NUM_LIT:0> <EOL> if n_block == first_block : <EOL> insert_offset = offset - block_offset <EOL> insert_end = self . blocksize <EOL> if n_block == last_block : <EOL> insert_end = end - block_offset <EOL> data_start = block_offset + insert_offset - offset <EOL> data_end = block_offset + insert_end - offset <EOL> log . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> n_block , insert_offset , insert_end , <EOL> data_start , data_end ) <EOL> current_data = self . read_block ( n_block ) <EOL> datafile = StringIO ( ) <EOL> datafile . write ( current_data ) <EOL> datafile . seek ( insert_offset ) <EOL> datafile . write ( data [ data_start : data_end ] ) <EOL> self . write_block ( n_block , datafile . getvalue ( ) ) <EOL> if end > current_size : <EOL> self [ '<STR_LIT:size>' ] = end <EOL> def truncate ( self , new_size ) : <EOL> log . info ( "<STR_LIT>" , repr ( self . name ) , new_size ) <EOL> current_size = self [ '<STR_LIT:size>' ] <EOL> if current_size < new_size : <EOL> self . write_data ( '<STR_LIT>' * ( new_size - current_size ) , current_size ) <EOL> elif current_size > new_size : <EOL> first_block = new_size / self . blocksize <EOL> last_block = current_size / self . blocksize <EOL> truncate_offset = new_size % self . blocksize <EOL> for n_block in range ( first_block , last_block + <NUM_LIT:1> ) : <EOL> if n_block == first_block and truncate_offset > <NUM_LIT:0> : <EOL> old_data = self . read_block ( n_block ) <EOL> self . write_block ( n_block , old_data [ : truncate_offset ] ) <EOL> else : <EOL> self . delete_block ( n_block ) <EOL> self [ '<STR_LIT:size>' ] = new_size <EOL> def unlink ( self ) : <EOL> log . info ( '<STR_LIT>' , self . name ) <EOL> nlink = self [ '<STR_LIT>' ] - <NUM_LIT:1> <EOL> if nlink > <NUM_LIT:0> : <EOL> log . info ( '<STR_LIT>' , self . name , nlink ) <EOL> self [ '<STR_LIT>' ] = nlink <EOL> else : <EOL> log . info ( '<STR_LIT>' , self . name ) <EOL> self . storage . _remove_inode ( self . name ) <EOL> self . tree . remove ( ) <EOL> self . storage . _autocommit ( ) <EOL> class StorageFile ( object ) : <EOL> is_dir = False <EOL> def __init__ ( self , name , inode , parent ) : <EOL> self . name = name <EOL> self . inode = inode <EOL> self . parent = parent <EOL> @ property <EOL> def path ( self ) : <EOL> return self . parent . path + self . name <EOL> @ property <EOL> def size ( self ) : <EOL> return self . inode [ '<STR_LIT:size>' ] <EOL> def _read_all_data ( self ) : <EOL> return self . read_data ( <NUM_LIT:0> , self . size ) <EOL> def read_data ( self , offset , length ) : <EOL> return self . inode . read_data ( offset , length ) <EOL> def write_data ( self , data , offset ) : <EOL> return self . inode . write_data ( data , offset ) <EOL> def truncate ( self , new_size ) : <EOL> return self . inode . truncate ( new_size ) <EOL> def unlink ( self ) : <EOL> log . info ( '<STR_LIT>' , repr ( self . path ) ) <EOL> self . parent . remove_ls_entry ( self . name ) <EOL> self . inode . unlink ( ) <EOL> def quote ( name ) : <EOL> return ( binascii . b2a_qp ( name , quotetabs = True , istext = False ) <EOL> . replace ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> unquote = binascii . a2b_qp <EOL> def check_filename ( name ) : <EOL> if name in ( '<STR_LIT:.>' , '<STR_LIT:..>' , '<STR_LIT>' ) or '<STR_LIT:/>' in name or len ( name ) > <NUM_LIT:255> : <EOL> raise ValueError ( "<STR_LIT>" % name ) <EOL> def iter_entries ( ls_data ) : <EOL> for line in ls_data . split ( '<STR_LIT:\n>' ) : <EOL> if not line : <EOL> continue <EOL> name , value = line . rsplit ( '<STR_LIT:U+0020>' , <NUM_LIT:1> ) <EOL> yield unquote ( name ) , value <EOL> upgrade_log = logging . getLogger ( '<STR_LIT>' ) <EOL> upgrade_log . setLevel ( logging . DEBUG ) <EOL> def storage_format_upgrade ( upgrade_name , upgrade_from , upgrade_to ) : <EOL> def decorator ( the_upgrade ) : <EOL> @ functools . wraps ( the_upgrade ) <EOL> def wrapper ( repo_path ) : <EOL> eg = EasyGit . open_repo ( repo_path ) <EOL> if '<STR_LIT>' not in eg . root : <EOL> upgrade_log . info ( '<STR_LIT>' , <EOL> repo_path ) <EOL> b = eg . root . new_blob ( '<STR_LIT>' ) <EOL> b . data = '<STR_LIT:{}>' <EOL> features = FeatureBlob ( eg . root [ '<STR_LIT>' ] ) <EOL> for name , value in upgrade_from . iteritems ( ) : <EOL> if features . get ( name , None ) is not value : <EOL> upgrade_log . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> upgrade_name , repo_path , name , value ) <EOL> return <EOL> upgrade_log . info ( '<STR_LIT>' , <EOL> upgrade_name , repo_path ) <EOL> the_upgrade ( eg ) <EOL> upgrade_log . debug ( '<STR_LIT>' , upgrade_name ) <EOL> for name , value in upgrade_to . iteritems ( ) : <EOL> features [ name ] = value <EOL> message = "<STR_LIT>" % upgrade_name <EOL> eg . commit ( GitStorage . commit_author , message , <EOL> [ eg . get_head_id ( '<STR_LIT>' ) ] ) <EOL> upgrade_log . info ( '<STR_LIT>' , <EOL> upgrade_name , repo_path ) <EOL> return wrapper <EOL> return decorator <EOL> @ storage_format_upgrade ( '<STR_LIT>' , <EOL> upgrade_from = { '<STR_LIT>' : None } , <EOL> upgrade_to = { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> def convert_fs_to_treetree_inodes ( eg ) : <EOL> """<STR_LIT>""" <EOL> inode_index = eg . root [ '<STR_LIT>' ] <EOL> class DummyStorage ( object ) : <EOL> def _autocommit ( self ) : pass <EOL> s = DummyStorage ( ) <EOL> for inode_name in inode_index : <EOL> upgrade_log . debug ( '<STR_LIT>' , inode_name ) <EOL> inode = StorageInode ( inode_name , inode_index [ inode_name ] , s ) <EOL> block_offsets = set ( ) <EOL> for old_block_name in inode . tree : <EOL> if old_block_name . startswith ( '<STR_LIT:b>' ) : <EOL> if not old_block_name . startswith ( '<STR_LIT>' ) : <EOL> block_offsets . add ( int ( old_block_name [ <NUM_LIT:1> : ] ) ) <EOL> for block_offset in sorted ( block_offsets ) : <EOL> old_block_name = '<STR_LIT>' % block_offset <EOL> new_block_name = str ( block_offset / StorageInode . blocksize ) <EOL> old_block = inode . tree [ old_block_name ] <EOL> new_block = inode . tt . clone ( old_block , new_block_name ) <EOL> del inode . tree [ old_block_name ] <EOL> inode [ '<STR_LIT:size>' ] = block_offset + len ( new_block . data ) <EOL> inode . tree . _commit ( ) <EOL> @ storage_format_upgrade ( '<STR_LIT>' , <EOL> upgrade_from = { '<STR_LIT>' : None } , <EOL> upgrade_to = { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> def convert_fs_to_treetree_inode_index ( eg ) : <EOL> """<STR_LIT>""" <EOL> inode_index_raw = eg . root [ '<STR_LIT>' ] <EOL> inode_index_tt = TreeTree ( inode_index_raw , prefix = '<STR_LIT>' ) <EOL> all_inode_names = list ( inode_index_raw . keys ( ) ) <EOL> largest_number = - <NUM_LIT:1> <EOL> for inode_name in all_inode_names : <EOL> upgrade_log . debug ( '<STR_LIT>' , inode_name ) <EOL> inode_index_tt . clone ( inode_index_raw [ inode_name ] , inode_name [ <NUM_LIT:1> : ] ) <EOL> del inode_index_raw [ inode_name ] <EOL> number = int ( inode_name [ <NUM_LIT:1> : ] ) <EOL> largest_number = max ( largest_number , number ) <EOL> FeatureBlob ( eg . root [ '<STR_LIT>' ] ) [ '<STR_LIT>' ] = largest_number + <NUM_LIT:1> <EOL> all_updates = [ <EOL> convert_fs_to_treetree_inodes , <EOL> convert_fs_to_treetree_inode_index , <EOL> ] </s>
