<s> """<STR_LIT>""" <EOL> import os <EOL> import hashlib <EOL> import logging <EOL> import subprocess <EOL> from cStringIO import StringIO <EOL> import requests <EOL> from modularodm import Q <EOL> from framework . transactions . context import TokuTransaction <EOL> from website import settings <EOL> from website . app import init_app <EOL> from website . models import Node <EOL> from website . addons . osffiles . model import NodeFile <EOL> from website . addons . osfstorage import model <EOL> from website . addons . osfstorage import utils <EOL> from website . addons . osfstorage import errors <EOL> from scripts import utils as script_utils <EOL> from scripts . osfstorage . utils import ensure_osf_files <EOL> from scripts . osfstorage import settings as scripts_settings <EOL> logger = logging . getLogger ( __name__ ) <EOL> script_utils . add_file_logger ( logger , __file__ ) <EOL> logging . basicConfig ( level = logging . INFO ) <EOL> client = scripts_settings . STORAGE_CLIENT_CLASS ( <EOL> ** scripts_settings . STORAGE_CLIENT_OPTIONS <EOL> ) <EOL> container = client . create_container ( scripts_settings . STORAGE_CONTAINER_NAME ) <EOL> class SizeMismatchError ( Exception ) : <EOL> pass <EOL> class HashMismatchError ( Exception ) : <EOL> pass <EOL> def check_node ( node ) : <EOL> """<STR_LIT>""" <EOL> if not node . files_current : <EOL> return True <EOL> try : <EOL> with open ( os . devnull , '<STR_LIT:w>' ) as fnull : <EOL> subprocess . check_call ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' ] , <EOL> cwd = os . path . join ( settings . UPLOADS_PATH , node . _id ) , <EOL> stdout = fnull , <EOL> stderr = fnull , <EOL> ) <EOL> return True <EOL> except subprocess . CalledProcessError : <EOL> return False <EOL> except OSError : <EOL> return False <EOL> def get_source_node ( node ) : <EOL> """<STR_LIT>""" <EOL> source = node . registered_from or node . forked_from <EOL> if source is None : <EOL> return None <EOL> if check_node ( source ) : <EOL> return source <EOL> return get_source_node ( source ) <EOL> def migrate_version ( idx , node_file , node_settings , node = None , dry_run = True ) : <EOL> """<STR_LIT>""" <EOL> node = node or node_settings . owner <EOL> logger . info ( '<STR_LIT>' . format ( <EOL> idx , <EOL> node_file . _id , <EOL> node . _id , <EOL> ) ) <EOL> if not dry_run : <EOL> record = model . OsfStorageFileRecord . get_or_create ( node_file . path , node_settings ) <EOL> if len ( record . versions ) > idx : <EOL> return <EOL> content = scripts_settings . SPECIAL_CASES . get ( ( node . _id , node_file . _id ) ) <EOL> if content is None : <EOL> content , _ = node . read_file_object ( node_file ) <EOL> logger . info ( '<STR_LIT>' . format ( len ( content ) , content [ : <NUM_LIT:10> ] ) ) <EOL> if dry_run : <EOL> return <EOL> md5 = hashlib . md5 ( content ) . hexdigest ( ) <EOL> file_pointer = StringIO ( content ) <EOL> hash_str = scripts_settings . UPLOAD_PRIMARY_HASH ( content ) . hexdigest ( ) <EOL> obj = container . get_or_upload_file ( file_pointer , hash_str ) <EOL> if obj . size != len ( content ) : <EOL> raise SizeMismatchError <EOL> if obj . md5 != md5 : <EOL> raise HashMismatchError <EOL> metadata = { <EOL> '<STR_LIT:size>' : obj . size , <EOL> '<STR_LIT>' : obj . content_type , <EOL> '<STR_LIT>' : obj . date_modified . isoformat ( ) , <EOL> '<STR_LIT>' : md5 , <EOL> } <EOL> try : <EOL> record . create_pending_version ( node_file . uploader , '<STR_LIT>' . format ( hash_str , idx ) ) <EOL> except errors . OsfStorageError : <EOL> latest_version = record . get_version ( required = True ) <EOL> record . remove_version ( latest_version ) <EOL> record = OsfStorageFileRecord . get_or_create ( node_file . path , node_settings ) <EOL> record . create_pending_version ( node_file . uploader , '<STR_LIT>' . format ( hash_str , idx ) ) <EOL> record . resolve_pending_version ( <EOL> '<STR_LIT>' . format ( hash_str , idx ) , <EOL> obj . location , <EOL> metadata , <EOL> log = False , <EOL> ) <EOL> def migrate_node ( node , dry_run = True ) : <EOL> """<STR_LIT>""" <EOL> logger . info ( '<STR_LIT>' . format ( node . _id ) ) <EOL> node_settings = node . get_or_add_addon ( '<STR_LIT>' , auth = None , log = False ) <EOL> repo_intact = check_node ( node ) <EOL> source_node = None <EOL> if not repo_intact : <EOL> logger . warn ( '<STR_LIT>' . format ( node . _id ) ) <EOL> source_node = get_source_node ( node ) <EOL> if source_node is None : <EOL> logger . error ( '<STR_LIT>' . format ( node . _id ) ) <EOL> for path , versions in node . files_versions . iteritems ( ) : <EOL> for idx , version in enumerate ( versions ) : <EOL> try : <EOL> node_file = NodeFile . load ( version ) <EOL> migrate_version ( idx , node_file , node_settings , node = source_node , dry_run = dry_run ) <EOL> except Exception as error : <EOL> logger . error ( '<STR_LIT>' . format ( version , node . _id ) ) <EOL> logger . exception ( error ) <EOL> break <EOL> def get_nodes ( ) : <EOL> return Node . find ( Q ( '<STR_LIT>' , '<STR_LIT>' , None ) ) <EOL> def main ( dry_run = True ) : <EOL> nodes = get_nodes ( ) <EOL> logger . info ( '<STR_LIT>' . format ( len ( nodes ) ) ) <EOL> for node in nodes : <EOL> try : <EOL> with TokuTransaction ( ) : <EOL> migrate_node ( node , dry_run = dry_run ) <EOL> except Exception as error : <EOL> logger . error ( '<STR_LIT>' . format ( node . _id ) ) <EOL> logger . exception ( error ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> import sys <EOL> dry_run = '<STR_LIT>' in sys . argv <EOL> ensure_osf_files ( settings ) <EOL> init_app ( set_backends = True , routes = False ) <EOL> main ( dry_run = dry_run ) <EOL> ensure_osf_files ( settings ) <EOL> from nose . tools import * <EOL> from tests . base import OsfTestCase <EOL> from tests . factories import ProjectFactory <EOL> import shutil <EOL> from framework . auth import Auth <EOL> class TestMigrateFiles ( OsfTestCase ) : <EOL> def setUp ( self ) : <EOL> super ( TestMigrateFiles , self ) . setUp ( ) <EOL> self . project = ProjectFactory ( ) <EOL> self . user = self . project . creator <EOL> self . auth_obj = Auth ( user = self . user ) <EOL> self . project . delete_addon ( '<STR_LIT>' , auth = None , _force = True ) <EOL> for idx in range ( <NUM_LIT:5> ) : <EOL> content = '<STR_LIT>' . format ( idx ) <EOL> self . project . add_file ( <EOL> auth = self . auth_obj , <EOL> file_name = '<STR_LIT>' , <EOL> content = content , <EOL> size = len ( content ) , <EOL> content_type = '<STR_LIT>' , <EOL> ) <EOL> def check_record ( self , record ) : <EOL> assert_true ( record ) <EOL> assert_equal ( len ( record . versions ) , <NUM_LIT:5> ) <EOL> for idx , version in enumerate ( record . versions ) : <EOL> assert_false ( version . pending ) <EOL> expected = '<STR_LIT>' . format ( idx ) <EOL> download_url = utils . get_waterbutler_download_url ( idx + <NUM_LIT:1> , version , record ) <EOL> resp = requests . get ( download_url ) <EOL> assert_equal ( expected , resp . content ) <EOL> def test_migrate ( self ) : <EOL> main ( dry_run = False ) <EOL> node_settings = self . project . get_addon ( '<STR_LIT>' ) <EOL> assert_true ( node_settings ) <EOL> record = model . OsfStorageFileRecord . find_by_path ( '<STR_LIT>' , node_settings ) <EOL> self . check_record ( record ) <EOL> main ( dry_run = False ) <EOL> assert_equal ( len ( record . versions ) , <NUM_LIT:5> ) <EOL> def test_migrate_incomplete ( self ) : <EOL> node_settings = self . project . get_or_add_addon ( '<STR_LIT>' , auth = None , log = False ) <EOL> record = model . OsfStorageFileRecord . get_or_create ( '<STR_LIT>' , node_settings ) <EOL> node_file = NodeFile . load ( self . project . files_versions [ '<STR_LIT>' ] [ <NUM_LIT:0> ] ) <EOL> content , _ = self . project . read_file_object ( node_file ) <EOL> file_pointer = StringIO ( content ) <EOL> hash_str = scripts_settings . UPLOAD_PRIMARY_HASH ( content ) . hexdigest ( ) <EOL> record . create_pending_version ( node_file . uploader , hash_str ) <EOL> main ( dry_run = False ) <EOL> def test_migrate_fork ( self ) : <EOL> fork = self . project . fork_node ( auth = self . auth_obj ) <EOL> main ( dry_run = False ) <EOL> node_settings = self . project . get_addon ( '<STR_LIT>' ) <EOL> record = model . OsfStorageFileRecord . find_by_path ( '<STR_LIT>' , node_settings ) <EOL> self . check_record ( record ) <EOL> fork_node_settings = fork . get_addon ( '<STR_LIT>' ) <EOL> fork_record = model . OsfStorageFileRecord . find_by_path ( '<STR_LIT>' , fork_node_settings ) <EOL> self . check_record ( fork_record ) <EOL> def test_migrate_corrupt_fork_repo_deleted ( self ) : <EOL> fork = self . project . fork_node ( auth = self . auth_obj ) <EOL> fork_repo = os . path . join ( settings . UPLOADS_PATH , fork . _id ) <EOL> shutil . rmtree ( fork_repo ) <EOL> main ( dry_run = False ) <EOL> node_settings = self . project . get_addon ( '<STR_LIT>' ) <EOL> record = model . OsfStorageFileRecord . find_by_path ( '<STR_LIT>' , node_settings ) <EOL> self . check_record ( record ) <EOL> fork_node_settings = fork . get_addon ( '<STR_LIT>' ) <EOL> fork_record = model . OsfStorageFileRecord . find_by_path ( '<STR_LIT>' , fork_node_settings ) <EOL> self . check_record ( fork_record ) <EOL> def test_migrate_corrupt_fork_git_dir_deleted ( self ) : <EOL> fork = self . project . fork_node ( auth = self . auth_obj ) <EOL> fork_git_dir = os . path . join ( settings . UPLOADS_PATH , fork . _id , '<STR_LIT>' ) <EOL> shutil . rmtree ( fork_git_dir ) <EOL> main ( dry_run = False ) <EOL> node_settings = self . project . get_addon ( '<STR_LIT>' ) <EOL> record = model . OsfStorageFileRecord . find_by_path ( '<STR_LIT>' , node_settings ) <EOL> self . check_record ( record ) <EOL> fork_node_settings = fork . get_addon ( '<STR_LIT>' ) <EOL> fork_record = model . OsfStorageFileRecord . find_by_path ( '<STR_LIT>' , fork_node_settings ) <EOL> self . check_record ( fork_record ) </s>
