original_docstring: Parses command line steps/pipeline keywords and returns list of steps to run.
         Step tuples include name of step, functions associated with step, and whether step can use existing products
docstring: Parses command line steps/pipeline keywords and returns list of steps to run.
  Step tuples include name of step, functions associated with step, and whether step can use existing products
docstring_tokens: Parses command line steps / pipeline keywords and returns list of steps to run . Step tuples include name of step functions associated with step and whether step can use existing products
code: def parse_steps(cli_args):
      '''Parses command line steps/pipeline keywords and returns list of steps to run.
         Step tuples include name of step, functions associated with step, and whether step can use existing products'''
      step_names = cli_args.steps
      use_existing = cli_args.use_existing
      predict_model = cli_args.predict_model
      train_feats = cli_args.train_feats
      field_mode = cli_args.field_mode
  
      cache_allowed_steps = [PREPROC_STEP, VALIDATE_STEP, TRACKER_STEP, POINT_EVAL_STEP, TRACK_EVAL_STEP]
  
      step_mappings = {
          PREPROC_STEP :      [preproc_experiment, None, None],
          VALIDATE_STEP :     [validate_experiment, validate_batch, None],
          PROJ_TRACKER_STEP : [proj_tracker_experiment, None, None],
          TRACKER_STEP :      [tracker_experiment, None, None],
          POINT_EVAL_STEP :   [point_eval_experiment, point_eval_batch, point_eval_load_cached],
          TRACK_EVAL_STEP :   [track_eval_experiment, track_eval_batch, track_eval_load_cached],
          FEATURES_STEP :     [features_experiment, None, None],
          TRAIN_STEP :        [None, train_batch, None],
          PREDICT_STEP :      [predict_experiment, predict_batch, None],
          ASDP_STEP :         [asdp_experiment, asdp_batch, None],
          MANIFEST_STEP :     [manifest_experiment, None, None]
      }
  
      pipelines = {
          PIPELINE_TRAIN :        [PREPROC_STEP, VALIDATE_STEP, TRACKER_STEP, TRACK_EVAL_STEP, FEATURES_STEP, TRAIN_STEP],
          PIPELINE_PREDICT :      [PREPROC_STEP, VALIDATE_STEP, TRACKER_STEP, FEATURES_STEP, PREDICT_STEP],
          PIPELINE_TRACKER_EVAL : [PREPROC_STEP, VALIDATE_STEP, TRACKER_STEP, POINT_EVAL_STEP, TRACK_EVAL_STEP],
          PIPELINE_PRODUCTS :     [PREPROC_STEP, VALIDATE_STEP, TRACKER_STEP, POINT_EVAL_STEP,
                                   TRACK_EVAL_STEP, FEATURES_STEP, PREDICT_STEP,
                                   ASDP_STEP, MANIFEST_STEP],
          PIPELINE_FIELD :        [PREPROC_STEP, VALIDATE_STEP, TRACKER_STEP, FEATURES_STEP,
                                   PREDICT_STEP, ASDP_STEP, MANIFEST_STEP]
      }
  
      # Pipeline-wise check
      if PIPELINE_FIELD in step_names and not field_mode:
          logging.error("--steps pipeline_field requires --field_mode")
  
      # Convert pipelines to steps
      if len(step_names) == 1 and step_names[0] in pipelines:
          step_names = pipelines[step_names[0]]
  
      # Various checks after substituting pipeline keywords
      if PREDICT_STEP in step_names and predict_model == "":
          logging.error("--steps predict requires --predict_model")
  
      if TRAIN_STEP in step_names and not train_feats:
          logging.error("--steps train requires --train_feats")
  
      if PREDICT_STEP in step_names and train_feats:
          logging.error("--steps predict shouldn't use --train_feats")
  
  
      # Create step tuples of the form: (step_name, exp_func, batch_func, get_previous_func, can_reuse)
      step_tuples = []
      for step_name in step_names:
          if step_name not in step_mappings:
              raise Exception("Unrecognized step or pipeline keyword: {}".format(step_name))
          step = [step_name]
          step.extend(step_mappings[step_name])
          step.append(step_name in use_existing and step_name in cache_allowed_steps)
          step_tuples.append(tuple(step))
  
      can_reuse = []
      for st in step_tuples:
          if st[-1]:
              can_reuse.append(st[0])
      if len(can_reuse) > 1:
          logging.info("USE EXISTING ENABLED FOR STEPS: {}".format(' '.join(can_reuse)))
  
      return step_tuples
code_tokens: def parse_steps ( cli_args ) : step_names = cli_args . steps use_existing = cli_args . use_existing predict_model = cli_args . predict_model train_feats = cli_args . train_feats field_mode = cli_args . field_mode cache_allowed_steps = [ PREPROC_STEP , VALIDATE_STEP , TRACKER_STEP , POINT_EVAL_STEP , TRACK_EVAL_STEP ] step_mappings = { PREPROC_STEP : [ preproc_experiment , None , None ] , VALIDATE_STEP : [ validate_experiment , validate_batch , None ] , PROJ_TRACKER_STEP : [ proj_tracker_experiment , None , None ] , TRACKER_STEP : [ tracker_experiment , None , None ] , POINT_EVAL_STEP : [ point_eval_experiment , point_eval_batch , point_eval_load_cached ] , TRACK_EVAL_STEP : [ track_eval_experiment , track_eval_batch , track_eval_load_cached ] , FEATURES_STEP : [ features_experiment , None , None ] , TRAIN_STEP : [ None , train_batch , None ] , PREDICT_STEP : [ predict_experiment , predict_batch , None ] , ASDP_STEP : [ asdp_experiment , asdp_batch , None ] , MANIFEST_STEP : [ manifest_experiment , None , None ] } pipelines = { PIPELINE_TRAIN : [ PREPROC_STEP , VALIDATE_STEP , TRACKER_STEP , TRACK_EVAL_STEP , FEATURES_STEP , TRAIN_STEP ] , PIPELINE_PREDICT : [ PREPROC_STEP , VALIDATE_STEP , TRACKER_STEP , FEATURES_STEP , PREDICT_STEP ] , PIPELINE_TRACKER_EVAL : [ PREPROC_STEP , VALIDATE_STEP , TRACKER_STEP , POINT_EVAL_STEP , TRACK_EVAL_STEP ] , PIPELINE_PRODUCTS : [ PREPROC_STEP , VALIDATE_STEP , TRACKER_STEP , POINT_EVAL_STEP , TRACK_EVAL_STEP , FEATURES_STEP , PREDICT_STEP , ASDP_STEP , MANIFEST_STEP ] , PIPELINE_FIELD : [ PREPROC_STEP , VALIDATE_STEP , TRACKER_STEP , FEATURES_STEP , PREDICT_STEP , ASDP_STEP , MANIFEST_STEP ] } if PIPELINE_FIELD in step_names and not field_mode : logging . error ( "--steps pipeline_field requires --field_mode" ) if len ( step_names ) == 1 and step_names [ 0 ] in pipelines : step_names = pipelines [ step_names [ 0 ] ] if PREDICT_STEP in step_names and predict_model == "" : logging . error ( "--steps predict requires --predict_model" ) if TRAIN_STEP in step_names and not train_feats : logging . error ( "--steps train requires --train_feats" ) if PREDICT_STEP in step_names and train_feats : logging . error ( "--steps predict shouldn't use --train_feats" ) step_tuples = [ ] for step_name in step_names : if step_name not in step_mappings : raise Exception ( "Unrecognized step or pipeline keyword: {}" . format ( step_name ) ) step = [ step_name ] step . extend ( step_mappings [ step_name ] ) step . append ( step_name in use_existing and step_name in cache_allowed_steps ) step_tuples . append ( tuple ( step ) ) can_reuse = [ ] for st in step_tuples : if st [ - 1 ] : can_reuse . append ( st [ 0 ] ) if len ( can_reuse ) > 1 : logging . info ( "USE EXISTING ENABLED FOR STEPS: {}" . format ( ' ' . join ( can_reuse ) ) ) return step_tuples
short_docstring: Parses command line steps/pipeline keywords and returns list of steps to run.
short_docstring_tokens: Parses command line steps / pipeline keywords and returns list of steps to run .
comment: '''Parses command line steps/pipeline keywords and returns list of steps to run.
         Step tuples include name of step, functions associated with step, and whether step can use existing products'''$SEP$# Pipeline-wise check$SEP$# Convert pipelines to steps$SEP$# Various checks after substituting pipeline keywords$SEP$# Create step tuples of the form: (step_name, exp_func, batch_func, get_previous_func, can_reuse)