<s> """<STR_LIT>""" <EOL> import os , sys , copy , fnmatch , re , shutil <EOL> import yaml , json , tempfile , mimetypes <EOL> import webbrowser , traceback , collections <EOL> import subprocess , string , random , pipes <EOL> from collections import OrderedDict <EOL> import shelve , getpass <EOL> from datetime import datetime <EOL> from hashlib import sha256 <EOL> import mimetypes <EOL> import platform <EOL> import uuid , shutil <EOL> import boto3 , glob2 <EOL> import subprocess <EOL> from dateutil import parser <EOL> try : <EOL> from urllib . parse import urlparse <EOL> except : <EOL> from urlparse import urlparse <EOL> from . . config import get_config <EOL> from . . plugins . common import plugins_get_mgr <EOL> from . . helper import bcolors , clean_str , cd , compute_sha256 , run , clean_name , log_repo_action <EOL> from . . exceptions import * <EOL> from . history import get_history , get_diffs <EOL> from . validation import validate <EOL> __all__ = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:status>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' <EOL> ] <EOL> def lookup ( username , reponame ) : <EOL> """<STR_LIT>""" <EOL> mgr = plugins_get_mgr ( ) <EOL> repomgr = mgr . get ( what = '<STR_LIT>' , name = '<STR_LIT>' ) <EOL> repo = repomgr . lookup ( username = username , <EOL> reponame = reponame ) <EOL> return repo <EOL> def list_repos ( remote = False ) : <EOL> """<STR_LIT>""" <EOL> mgr = plugins_get_mgr ( ) <EOL> if not remote : <EOL> repomgr = mgr . get ( what = '<STR_LIT>' , name = '<STR_LIT>' ) <EOL> repos = repomgr . get_repo_list ( ) <EOL> repos . sort ( ) <EOL> return repos <EOL> else : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> @ log_repo_action <EOL> def shellcmd ( repo , args ) : <EOL> """<STR_LIT>""" <EOL> with cd ( repo . rootdir ) : <EOL> result = run ( args ) <EOL> return result <EOL> def datapackage_exists ( repo ) : <EOL> """<STR_LIT>""" <EOL> datapath = os . path . join ( repo . rootdir , "<STR_LIT>" ) <EOL> return os . path . exists ( datapath ) <EOL> def generic_repo_cmd ( repo , cmd , * args ) : <EOL> return repo . run ( cmd , * args ) <EOL> @ log_repo_action <EOL> def log ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> return generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> @ log_repo_action <EOL> def show ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> return generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> @ log_repo_action <EOL> def remote ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> return generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> @ log_repo_action <EOL> def push ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> return generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> @ log_repo_action <EOL> def pull ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> return generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> @ log_repo_action <EOL> def commit ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> return generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> def drop ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> return generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> @ log_repo_action <EOL> def stash ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> return generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> @ log_repo_action <EOL> def diff ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> return generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> @ log_repo_action <EOL> def status ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> result = generic_repo_cmd ( repo , '<STR_LIT:status>' , args ) <EOL> return result <EOL> @ log_repo_action <EOL> def delete ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> result = generic_repo_cmd ( repo , '<STR_LIT>' , args ) <EOL> if result [ '<STR_LIT:status>' ] != '<STR_LIT:success>' : <EOL> return status <EOL> with cd ( repo . rootdir ) : <EOL> package = repo . package <EOL> resources = package [ '<STR_LIT>' ] <EOL> cleaned_resources = [ ] <EOL> for r in resources : <EOL> relativepath = r [ '<STR_LIT>' ] <EOL> sha256 = r [ '<STR_LIT>' ] <EOL> if relativepath not in [ '<STR_LIT>' , None ] : <EOL> if not os . path . exists ( relativepath ) : <EOL> print ( "<STR_LIT>" , relativepath ) <EOL> continue <EOL> cleaned_resources . append ( r ) <EOL> package [ '<STR_LIT>' ] = cleaned_resources <EOL> repo . package = package <EOL> with open ( '<STR_LIT>' , '<STR_LIT:w>' ) as fd : <EOL> fd . write ( json . dumps ( repo . package , indent = <NUM_LIT:4> ) ) <EOL> return { <EOL> '<STR_LIT:status>' : '<STR_LIT:success>' , <EOL> '<STR_LIT:message>' : '<STR_LIT>' <EOL> } <EOL> def bootstrap_datapackage ( repo , force = False , <EOL> options = None , noinput = False ) : <EOL> """<STR_LIT>""" <EOL> print ( "<STR_LIT>" ) <EOL> tsprefix = datetime . now ( ) . date ( ) . isoformat ( ) <EOL> package = OrderedDict ( [ <EOL> ( '<STR_LIT:title>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT:description>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT:username>' , repo . username ) , <EOL> ( '<STR_LIT>' , repo . reponame ) , <EOL> ( '<STR_LIT:name>' , str ( repo ) ) , <EOL> ( '<STR_LIT:title>' , "<STR_LIT>" ) , <EOL> ( '<STR_LIT:description>' , "<STR_LIT>" ) , <EOL> ( '<STR_LIT>' , [ ] ) , <EOL> ( '<STR_LIT>' , [ ] ) , <EOL> ( '<STR_LIT>' , getpass . getuser ( ) ) , <EOL> ( '<STR_LIT>' , datetime . now ( ) . isoformat ( ) ) , <EOL> ( '<STR_LIT>' , repo . remoteurl ) <EOL> ] ) <EOL> if options is not None : <EOL> package [ '<STR_LIT:title>' ] = options [ '<STR_LIT:title>' ] <EOL> package [ '<STR_LIT:description>' ] = options [ '<STR_LIT:description>' ] <EOL> else : <EOL> if noinput : <EOL> raise IncompleteParameters ( "<STR_LIT>" ) <EOL> for var in [ '<STR_LIT:title>' , '<STR_LIT:description>' ] : <EOL> value = '<STR_LIT>' <EOL> while value in [ '<STR_LIT>' , None ] : <EOL> value = input ( '<STR_LIT>' + var . title ( ) + "<STR_LIT>" ) <EOL> if len ( value ) == <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" . format ( var . title ( ) ) ) <EOL> package [ var ] = value <EOL> ( handle , filename ) = tempfile . mkstemp ( ) <EOL> with open ( filename , '<STR_LIT:w>' ) as fd : <EOL> fd . write ( json . dumps ( package , indent = <NUM_LIT:4> ) ) <EOL> repo . package = package <EOL> return filename <EOL> def init ( username , reponame , setup , <EOL> force = False , options = None , <EOL> noinput = False ) : <EOL> """<STR_LIT>""" <EOL> mgr = plugins_get_mgr ( ) <EOL> repomgr = mgr . get ( what = '<STR_LIT>' , name = '<STR_LIT>' ) <EOL> backendmgr = None <EOL> if setup == '<STR_LIT>' : <EOL> backendmgr = mgr . get ( what = '<STR_LIT>' , name = '<STR_LIT>' ) <EOL> repo = repomgr . init ( username , reponame , force , backendmgr ) <EOL> ( handle , gitignore ) = tempfile . mkstemp ( ) <EOL> with open ( gitignore , '<STR_LIT:w>' ) as fd : <EOL> fd . write ( "<STR_LIT>" ) <EOL> try : <EOL> filename = bootstrap_datapackage ( repo , force , options , noinput ) <EOL> except Exception as e : <EOL> repomgr . drop ( repo , [ ] ) <EOL> os . unlink ( gitignore ) <EOL> raise e <EOL> repo . run ( '<STR_LIT>' , <EOL> [ <EOL> { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : filename , <EOL> } , <EOL> { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : gitignore , <EOL> } , <EOL> ] ) <EOL> os . unlink ( filename ) <EOL> os . unlink ( gitignore ) <EOL> args = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> repo . run ( '<STR_LIT>' , args ) <EOL> return repo <EOL> def clone ( url ) : <EOL> """<STR_LIT>""" <EOL> backend = None <EOL> backendmgr = None <EOL> if url . startswith ( '<STR_LIT>' ) : <EOL> backendtype = '<STR_LIT>' <EOL> elif url . startswith ( "<STR_LIT:http>" ) or url . startswith ( "<STR_LIT>" ) : <EOL> backendtype = '<STR_LIT>' <EOL> else : <EOL> backendtype = None <EOL> mgr = plugins_get_mgr ( ) <EOL> repomgr = mgr . get ( what = '<STR_LIT>' , name = '<STR_LIT>' ) <EOL> backendmgr = mgr . get ( what = '<STR_LIT>' , name = backendtype ) <EOL> if backendmgr is not None and not backendmgr . url_is_valid ( url ) : <EOL> raise InvalidParameters ( "<STR_LIT>" ) <EOL> key = repomgr . clone ( url , backendmgr ) <EOL> repo = repomgr . lookup ( key = key ) <EOL> if not datapackage_exists ( repo ) : <EOL> filename = bootstrap_datapackage ( repo ) <EOL> repo . run ( '<STR_LIT>' , <EOL> [ <EOL> { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : filename , <EOL> } , <EOL> ] ) <EOL> os . unlink ( filename ) <EOL> args = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> repo . run ( '<STR_LIT>' , args ) <EOL> return repo <EOL> def annotate_metadata_data ( repo , task , patterns = [ "<STR_LIT:*>" ] , size = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> mgr = plugins_get_mgr ( ) <EOL> keys = mgr . search ( '<STR_LIT>' ) [ '<STR_LIT>' ] <EOL> representations = [ mgr . get_by_key ( '<STR_LIT>' , k ) for k in keys ] <EOL> matching_files = repo . find_matching_files ( patterns ) <EOL> package = repo . package <EOL> rootdir = repo . rootdir <EOL> files = package [ '<STR_LIT>' ] <EOL> for f in files : <EOL> relativepath = f [ '<STR_LIT>' ] <EOL> if relativepath in matching_files : <EOL> path = os . path . join ( rootdir , relativepath ) <EOL> if task == '<STR_LIT>' : <EOL> print ( "<STR_LIT>" , relativepath ) <EOL> f [ '<STR_LIT:content>' ] = open ( path ) . read ( ) [ : size ] <EOL> elif task == '<STR_LIT>' : <EOL> for r in representations : <EOL> if r . can_process ( path ) : <EOL> print ( "<STR_LIT>" , path ) <EOL> f [ '<STR_LIT>' ] = r . get_schema ( path ) <EOL> break <EOL> def annotate_metadata_code ( repo , files ) : <EOL> """<STR_LIT>""" <EOL> package = repo . package <EOL> package [ '<STR_LIT:code>' ] = [ ] <EOL> for p in files : <EOL> matching_files = glob2 . glob ( "<STR_LIT>" . format ( p ) ) <EOL> for f in matching_files : <EOL> absf = os . path . abspath ( f ) <EOL> print ( "<STR_LIT>" . format ( f ) ) <EOL> package [ '<STR_LIT:code>' ] . append ( OrderedDict ( [ <EOL> ( '<STR_LIT>' , f ) , <EOL> ( '<STR_LIT>' , repo . manager . permalink ( repo , absf ) ) , <EOL> ( '<STR_LIT>' , mimetypes . guess_type ( absf ) [ <NUM_LIT:0> ] ) , <EOL> ( '<STR_LIT>' , compute_sha256 ( absf ) ) <EOL> ] ) ) <EOL> def annotate_metadata_action ( repo ) : <EOL> """<STR_LIT>""" <EOL> package = repo . package <EOL> print ( "<STR_LIT>" ) <EOL> with cd ( repo . rootdir ) : <EOL> filename = "<STR_LIT>" <EOL> if os . path . exists ( filename ) : <EOL> history = open ( filename ) . readlines ( ) <EOL> actions = [ ] <EOL> for a in history : <EOL> try : <EOL> a = json . loads ( a ) <EOL> for x in [ '<STR_LIT:code>' ] : <EOL> if x not in a or a [ x ] == None : <EOL> a [ x ] = "<STR_LIT>" <EOL> actions . append ( a ) <EOL> except : <EOL> pass <EOL> package [ '<STR_LIT>' ] = actions <EOL> def annotate_metadata_platform ( repo ) : <EOL> """<STR_LIT>""" <EOL> print ( "<STR_LIT>" ) <EOL> package = repo . package <EOL> mgr = plugins_get_mgr ( ) <EOL> repomgr = mgr . get ( what = '<STR_LIT>' , name = '<STR_LIT>' ) <EOL> package [ '<STR_LIT>' ] = repomgr . get_metadata ( ) <EOL> def annotate_metadata_diffs ( repo ) : <EOL> print ( "<STR_LIT>" ) <EOL> with cd ( repo . rootdir ) : <EOL> get_diffs ( repo . package [ '<STR_LIT>' ] ) <EOL> def annotate_metadata_validation ( repo ) : <EOL> print ( "<STR_LIT>" ) <EOL> results = validate ( repo ) <EOL> fileresults = { } <EOL> for r in results : <EOL> filename = r [ '<STR_LIT:target>' ] <EOL> if filename not in fileresults : <EOL> fileresults [ filename ] = [ ] <EOL> fileresults [ filename ] . append ( r ) <EOL> for r in repo . package [ '<STR_LIT>' ] : <EOL> path = r [ '<STR_LIT>' ] <EOL> if path in fileresults : <EOL> r [ '<STR_LIT>' ] = fileresults [ path ] <EOL> def annotate_metadata_dependencies ( repo ) : <EOL> """<STR_LIT>""" <EOL> options = repo . options <EOL> if '<STR_LIT>' not in options : <EOL> print ( "<STR_LIT>" ) <EOL> return [ ] <EOL> repos = [ ] <EOL> dependent_repos = options [ '<STR_LIT>' ] <EOL> for d in dependent_repos : <EOL> if "<STR_LIT:/>" not in d : <EOL> print ( "<STR_LIT>" ) <EOL> ( username , reponame ) = d . split ( "<STR_LIT:/>" ) <EOL> try : <EOL> repos . append ( repo . manager . lookup ( username , reponame ) ) <EOL> except : <EOL> print ( "<STR_LIT>" , d ) <EOL> package = repo . package <EOL> package [ '<STR_LIT>' ] = [ ] <EOL> for r in repos : <EOL> package [ '<STR_LIT>' ] . append ( { <EOL> '<STR_LIT:username>' : r . username , <EOL> '<STR_LIT>' : r . reponame , <EOL> } ) <EOL> def post ( repo , args = [ ] ) : <EOL> """<STR_LIT>""" <EOL> mgr = plugins_get_mgr ( ) <EOL> keys = mgr . search ( what = '<STR_LIT>' ) <EOL> keys = keys [ '<STR_LIT>' ] <EOL> if len ( keys ) == <NUM_LIT:0> : <EOL> return <EOL> if '<STR_LIT>' in repo . options : <EOL> for name , details in repo . options [ '<STR_LIT>' ] . items ( ) : <EOL> patterns = details [ '<STR_LIT>' ] <EOL> matching_files = repo . find_matching_files ( patterns ) <EOL> matching_files . sort ( ) <EOL> details [ '<STR_LIT>' ] = matching_files <EOL> for i , f in enumerate ( matching_files ) : <EOL> r = repo . get_resource ( f ) <EOL> if '<STR_LIT>' not in r : <EOL> r [ '<STR_LIT>' ] = [ ] <EOL> r [ '<STR_LIT>' ] . append ( name + "<STR_LIT>" . format ( i ) ) <EOL> if '<STR_LIT>' in repo . options : <EOL> print ( "<STR_LIT>" ) <EOL> metadata = repo . options [ '<STR_LIT>' ] <EOL> if '<STR_LIT>' in metadata and metadata [ '<STR_LIT>' ] : <EOL> repo . package [ '<STR_LIT>' ] = get_history ( repo . rootdir ) <EOL> if '<STR_LIT>' in metadata and metadata [ '<STR_LIT>' ] : <EOL> annotate_metadata_action ( repo ) <EOL> if '<STR_LIT>' in metadata : <EOL> annotate_metadata_data ( repo , <EOL> task = '<STR_LIT>' , <EOL> patterns = metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> size = metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> if ( ( '<STR_LIT>' in metadata ) and metadata [ '<STR_LIT>' ] ) : <EOL> annotate_metadata_data ( repo , task = '<STR_LIT>' ) <EOL> if '<STR_LIT>' in metadata : <EOL> annotate_metadata_code ( repo , files = metadata [ '<STR_LIT>' ] ) <EOL> if '<STR_LIT>' in metadata : <EOL> annotate_metadata_platform ( repo ) <EOL> if '<STR_LIT>' in metadata : <EOL> annotate_metadata_validation ( repo ) <EOL> if '<STR_LIT>' in metadata : <EOL> annotate_metadata_dependencies ( repo ) <EOL> history = repo . package . get ( '<STR_LIT>' , None ) <EOL> if ( ( '<STR_LIT>' in metadata ) and <EOL> metadata [ '<STR_LIT>' ] and <EOL> history is not None ) : <EOL> annotate_metadata_diffs ( repo ) <EOL> repo . package [ '<STR_LIT>' ] = repo . options <EOL> try : <EOL> for k in keys : <EOL> metadatamgr = mgr . get_by_key ( '<STR_LIT>' , k ) <EOL> url = metadatamgr . url <EOL> o = urlparse ( url ) <EOL> print ( "<STR_LIT>" , o . netloc ) <EOL> response = metadatamgr . post ( repo ) <EOL> if isinstance ( response , str ) : <EOL> print ( "<STR_LIT>" , response ) <EOL> elif response . status_code in [ <NUM_LIT> ] : <EOL> content = response . json ( ) <EOL> print ( "<STR_LIT>" ) <EOL> for k in content : <EOL> print ( "<STR_LIT:U+0020>" , k , "<STR_LIT>" , "<STR_LIT:U+002C>" . join ( content [ k ] ) ) <EOL> except NetworkError as e : <EOL> print ( "<STR_LIT>" ) <EOL> except NetworkInvalidConfiguration as e : <EOL> print ( "<STR_LIT>" ) <EOL> print ( e . message ) <EOL> except Exception as e : <EOL> print ( "<STR_LIT>" ) <EOL> print ( e ) </s>
