<s> from functools import partial <EOL> from collections import Sequence <EOL> import os <EOL> from pathlib import Path <EOL> import random <EOL> from . . utils import _norm_path <EOL> from menpo . base import menpo_src_dir_path , LazyList <EOL> from menpo . visualize import print_progress <EOL> def data_dir_path ( ) : <EOL> r"""<STR_LIT>""" <EOL> return menpo_src_dir_path ( ) / '<STR_LIT:data>' <EOL> def data_path_to ( asset_filename ) : <EOL> r"""<STR_LIT>""" <EOL> asset_path = data_dir_path ( ) / asset_filename <EOL> if not asset_path . is_file ( ) : <EOL> raise ValueError ( "<STR_LIT>" . format ( <EOL> asset_filename , ls_builtin_assets ( ) ) ) <EOL> return asset_path <EOL> def same_name ( path ) : <EOL> r"""<STR_LIT>""" <EOL> pattern = path . with_suffix ( '<STR_LIT>' ) <EOL> return { p . suffix [ <NUM_LIT:1> : ] . upper ( ) : p for p in landmark_file_paths ( pattern ) } <EOL> def same_name_video ( path , frame_number ) : <EOL> r"""<STR_LIT>""" <EOL> pattern = path . with_name ( '<STR_LIT>' . format ( path . stem , frame_number ) ) <EOL> return { p . suffix [ <NUM_LIT:1> : ] . upper ( ) : p for p in landmark_file_paths ( pattern ) } <EOL> def import_image ( filepath , landmark_resolver = same_name , normalise = True ) : <EOL> r"""<STR_LIT>""" <EOL> kwargs = { '<STR_LIT>' : normalise } <EOL> return _import ( filepath , image_types , <EOL> landmark_ext_map = image_landmark_types , <EOL> landmark_resolver = landmark_resolver , <EOL> landmark_attach_func = _import_object_attach_landmarks , <EOL> importer_kwargs = kwargs ) <EOL> def import_video ( filepath , landmark_resolver = same_name_video , normalise = True , <EOL> importer_method = '<STR_LIT>' ) : <EOL> r"""<STR_LIT>""" <EOL> kwargs = { '<STR_LIT>' : normalise } <EOL> video_importer_methods = { '<STR_LIT>' : ffmpeg_video_types } <EOL> if importer_method not in video_importer_methods : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( video_importer_methods . keys ( ) ) ) <EOL> return _import ( filepath , video_importer_methods [ importer_method ] , <EOL> landmark_ext_map = image_landmark_types , <EOL> landmark_resolver = landmark_resolver , <EOL> landmark_attach_func = _import_lazylist_attach_landmarks , <EOL> importer_kwargs = kwargs ) <EOL> def import_landmark_file ( filepath , asset = None ) : <EOL> r"""<STR_LIT>""" <EOL> return _import ( filepath , image_landmark_types , asset = asset ) <EOL> def import_pickle ( filepath ) : <EOL> r"""<STR_LIT>""" <EOL> return _import ( filepath , pickle_types ) <EOL> def import_images ( pattern , max_images = None , shuffle = False , <EOL> landmark_resolver = same_name , normalise = True , <EOL> as_generator = False , verbose = False ) : <EOL> r"""<STR_LIT>""" <EOL> kwargs = { '<STR_LIT>' : normalise } <EOL> return _import_glob_lazy_list ( <EOL> pattern , image_types , <EOL> max_assets = max_images , shuffle = shuffle , <EOL> landmark_resolver = landmark_resolver , <EOL> landmark_ext_map = image_landmark_types , <EOL> landmark_attach_func = _import_object_attach_landmarks , <EOL> as_generator = as_generator , <EOL> verbose = verbose , <EOL> importer_kwargs = kwargs <EOL> ) <EOL> def import_videos ( pattern , max_videos = None , shuffle = False , <EOL> landmark_resolver = same_name_video , normalise = True , <EOL> importer_method = '<STR_LIT>' , as_generator = False , verbose = False ) : <EOL> r"""<STR_LIT>""" <EOL> kwargs = { '<STR_LIT>' : normalise } <EOL> video_importer_methods = { '<STR_LIT>' : ffmpeg_video_types } <EOL> if importer_method not in video_importer_methods : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( video_importer_methods . keys ( ) ) ) <EOL> return _import_glob_lazy_list ( <EOL> pattern , video_importer_methods [ importer_method ] , <EOL> max_assets = max_videos , shuffle = shuffle , <EOL> landmark_resolver = landmark_resolver , <EOL> landmark_ext_map = image_landmark_types , <EOL> landmark_attach_func = _import_lazylist_attach_landmarks , <EOL> as_generator = as_generator , <EOL> verbose = verbose , <EOL> importer_kwargs = kwargs <EOL> ) <EOL> def import_landmark_files ( pattern , max_landmarks = None , shuffle = False , <EOL> as_generator = False , verbose = False ) : <EOL> r"""<STR_LIT>""" <EOL> return _import_glob_lazy_list ( pattern , image_landmark_types , <EOL> max_assets = max_landmarks , shuffle = shuffle , <EOL> as_generator = as_generator , verbose = verbose ) <EOL> def import_pickles ( pattern , max_pickles = None , shuffle = False , as_generator = False , <EOL> verbose = False ) : <EOL> r"""<STR_LIT>""" <EOL> return _import_glob_lazy_list ( pattern , pickle_types , <EOL> max_assets = max_pickles , shuffle = shuffle , <EOL> as_generator = as_generator , verbose = verbose ) <EOL> def _import_builtin_asset ( asset_name , ** kwargs ) : <EOL> r"""<STR_LIT>""" <EOL> asset_path = data_path_to ( asset_name ) <EOL> try : <EOL> return _import ( asset_path , image_types , <EOL> landmark_ext_map = image_landmark_types , <EOL> landmark_attach_func = _import_object_attach_landmarks , <EOL> importer_kwargs = kwargs ) <EOL> except ValueError : <EOL> return _import ( asset_path , image_landmark_types , <EOL> importer_kwargs = kwargs ) <EOL> def ls_builtin_assets ( ) : <EOL> r"""<STR_LIT>""" <EOL> return [ p . name for p in data_dir_path ( ) . glob ( '<STR_LIT:*>' ) if not p . is_dir ( ) ] <EOL> def import_builtin ( x ) : <EOL> def execute ( ** kwargs ) : <EOL> return _import_builtin_asset ( x , ** kwargs ) <EOL> return execute <EOL> class BuiltinAssets ( object ) : <EOL> def __call__ ( self , asset_name , ** kwargs ) : <EOL> return _import_builtin_asset ( asset_name , ** kwargs ) <EOL> import_builtin_asset = BuiltinAssets ( ) <EOL> for asset in ls_builtin_assets ( ) : <EOL> setattr ( import_builtin_asset , asset . replace ( '<STR_LIT:.>' , '<STR_LIT:_>' ) , <EOL> import_builtin ( asset ) ) <EOL> def image_paths ( pattern ) : <EOL> r"""<STR_LIT>""" <EOL> return glob_with_suffix ( pattern , image_types ) <EOL> def video_paths ( pattern ) : <EOL> r"""<STR_LIT>""" <EOL> return glob_with_suffix ( pattern , ffmpeg_video_types ) <EOL> def landmark_file_paths ( pattern ) : <EOL> r"""<STR_LIT>""" <EOL> return glob_with_suffix ( pattern , image_landmark_types ) <EOL> def _import_glob_lazy_list ( pattern , extension_map , max_assets = None , <EOL> landmark_resolver = same_name , shuffle = False , <EOL> as_generator = False , landmark_ext_map = None , <EOL> landmark_attach_func = None , importer_kwargs = None , <EOL> verbose = False ) : <EOL> filepaths = list ( glob_with_suffix ( pattern , extension_map , <EOL> sort = ( not shuffle ) ) ) <EOL> if shuffle : <EOL> random . shuffle ( filepaths ) <EOL> if ( max_assets is not None ) and max_assets <= <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( max_assets ) ) <EOL> elif max_assets : <EOL> filepaths = filepaths [ : max_assets ] <EOL> n_files = len ( filepaths ) <EOL> if n_files == <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' . format ( pattern ) ) <EOL> lazy_list = LazyList ( [ partial ( _import , f , extension_map , <EOL> landmark_resolver = landmark_resolver , <EOL> landmark_ext_map = landmark_ext_map , <EOL> landmark_attach_func = landmark_attach_func , <EOL> importer_kwargs = importer_kwargs ) <EOL> for f in filepaths ] ) <EOL> if verbose and as_generator : <EOL> lazy_list = print_progress ( lazy_list , prefix = '<STR_LIT>' , <EOL> n_items = n_files ) <EOL> elif verbose : <EOL> print ( '<STR_LIT>' . format ( <EOL> n_files ) ) <EOL> if as_generator : <EOL> return ( a for a in lazy_list ) <EOL> else : <EOL> return lazy_list <EOL> def _import_object_attach_landmarks ( built_objects , landmark_resolver , <EOL> landmark_ext_map = None ) : <EOL> if landmark_ext_map is not None : <EOL> for x in built_objects : <EOL> lm_paths = landmark_resolver ( x . path ) <EOL> if lm_paths is None : <EOL> continue <EOL> for group_name , lm_path in lm_paths . items ( ) : <EOL> lms = _import ( lm_path , landmark_ext_map , asset = x ) <EOL> if x . n_dims == lms . n_dims : <EOL> x . landmarks [ group_name ] = lms <EOL> def _import_lazylist_attach_landmarks ( built_objects , landmark_resolver , <EOL> landmark_ext_map = None ) : <EOL> if landmark_ext_map is not None : <EOL> for k in range ( len ( built_objects ) ) : <EOL> x = built_objects [ k ] <EOL> lm_paths = partial ( landmark_resolver , x . path ) <EOL> def wrap_landmarks ( f , index ) : <EOL> obj = f ( ) <EOL> for group_name , lm_path in lm_paths ( index ) . items ( ) : <EOL> lms = _import ( lm_path , landmark_ext_map , asset = obj ) <EOL> if obj . n_dims == lms . n_dims : <EOL> obj . landmarks [ group_name ] = lms <EOL> return obj <EOL> new_ll = LazyList ( [ partial ( wrap_landmarks , c , i ) <EOL> for i , c in enumerate ( x . _callables ) ] ) <EOL> built_objects [ k ] = new_ll <EOL> def _import ( filepath , extensions_map , landmark_resolver = same_name , <EOL> landmark_ext_map = None , landmark_attach_func = None , <EOL> asset = None , importer_kwargs = None ) : <EOL> r"""<STR_LIT>""" <EOL> path = _norm_path ( filepath ) <EOL> if not path . is_file ( ) : <EOL> raise ValueError ( "<STR_LIT>" . format ( path ) ) <EOL> importer = importer_for_filepath ( path , extensions_map , <EOL> importer_kwargs = importer_kwargs ) <EOL> if asset is not None : <EOL> built_objects = importer . build ( asset = asset ) <EOL> else : <EOL> built_objects = importer . build ( ) <EOL> if not isinstance ( built_objects , list ) : <EOL> built_objects = [ built_objects ] <EOL> for x in built_objects : <EOL> if not hasattr ( x , '<STR_LIT:path>' ) : <EOL> try : <EOL> x . path = path <EOL> except AttributeError : <EOL> pass <EOL> if landmark_attach_func is not None : <EOL> landmark_attach_func ( built_objects , landmark_resolver , <EOL> landmark_ext_map = landmark_ext_map ) <EOL> if len ( built_objects ) == <NUM_LIT:1> : <EOL> built_objects = built_objects [ <NUM_LIT:0> ] <EOL> return built_objects <EOL> def _pathlib_glob_for_pattern ( pattern , sort = True ) : <EOL> r"""<STR_LIT>""" <EOL> pattern = _norm_path ( pattern ) <EOL> pattern_str = str ( pattern ) <EOL> gsplit = pattern_str . split ( '<STR_LIT:*>' , <NUM_LIT:1> ) <EOL> if len ( gsplit ) == <NUM_LIT:1> : <EOL> if Path ( pattern ) . is_dir ( ) : <EOL> preglob = pattern_str <EOL> pattern = '<STR_LIT:*>' <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( pattern ) ) <EOL> else : <EOL> preglob = gsplit [ <NUM_LIT:0> ] <EOL> pattern = '<STR_LIT:*>' + gsplit [ <NUM_LIT:1> ] <EOL> if not os . path . isdir ( preglob ) : <EOL> preglob , pattern_prefix = os . path . split ( preglob ) <EOL> pattern = pattern_prefix + pattern <EOL> p = Path ( preglob ) <EOL> paths = p . glob ( str ( pattern ) ) <EOL> if sort : <EOL> paths = sorted ( paths ) <EOL> return paths <EOL> def glob_with_suffix ( pattern , extensions_map , sort = True ) : <EOL> r"""<STR_LIT>""" <EOL> for path in _pathlib_glob_for_pattern ( pattern , sort = sort ) : <EOL> if ( '<STR_LIT>' . join ( path . suffixes [ - <NUM_LIT:2> : ] ) in extensions_map or <EOL> path . suffix in extensions_map ) : <EOL> yield path <EOL> def importer_for_filepath ( filepath , extensions_map , importer_kwargs = None ) : <EOL> r"""<STR_LIT>""" <EOL> suffix = '<STR_LIT>' . join ( filepath . suffixes ) <EOL> if suffix . isupper ( ) : <EOL> suffix = suffix . lower ( ) <EOL> importer_type = extensions_map . get ( suffix ) <EOL> if importer_type is None and len ( filepath . suffixes ) > <NUM_LIT:1> : <EOL> suffix = filepath . suffix <EOL> importer_type = extensions_map . get ( suffix ) <EOL> if importer_type is None : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" . format ( suffix ) ) <EOL> if importer_kwargs is not None : <EOL> return importer_type ( str ( filepath ) , ** importer_kwargs ) <EOL> else : <EOL> return importer_type ( str ( filepath ) ) <EOL> class Importer ( object ) : <EOL> r"""<STR_LIT>""" <EOL> def __init__ ( self , filepath ) : <EOL> self . filepath = os . path . abspath ( os . path . expanduser ( filepath ) ) <EOL> self . filename = os . path . splitext ( os . path . basename ( self . filepath ) ) [ <NUM_LIT:0> ] <EOL> self . extension = os . path . splitext ( self . filepath ) [ <NUM_LIT:1> ] <EOL> self . folder = os . path . dirname ( self . filepath ) <EOL> def build ( self ) : <EOL> r"""<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> from menpo . io . input . extensions import ( image_landmark_types , image_types , <EOL> pickle_types , ffmpeg_video_types ) </s>
