original_docstring: A Estimator fit() call to start a model evaluation job.
  
          Args:
              input_data_s3_prefix (str): Defines the location of s3 data used for evaluation
              manifest_file_path (str): Manifest file used to provide evaluation data.
              evaluation_job_name (str): Unique Sagemaker job name to identify the evaluation job
              local_mode (bool): Whether the evaluation job is running on local mode
              wait (bool): Whether the call should wait until the job completes. Only
                  meaningful when running in SageMaker mode.
              logs (bool): Whether to show the logs produced by the job.
                  Only meaningful when wait is True.
          
docstring: A Estimator fit() call to start a model evaluation job.
docstring_tokens: A Estimator fit () call to start a model evaluation job .
code: def evaluate(
          self,
          input_data_s3_prefix=None,
          manifest_file_path=None,
          evaluation_job_name=None,
          local_mode=True,
          wait=False,
          logs=True,
      ):
          """A Estimator fit() call to start a model evaluation job.
  
          Args:
              input_data_s3_prefix (str): Defines the location of s3 data used for evaluation
              manifest_file_path (str): Manifest file used to provide evaluation data.
              evaluation_job_name (str): Unique Sagemaker job name to identify the evaluation job
              local_mode (bool): Whether the evaluation job is running on local mode
              wait (bool): Whether the call should wait until the job completes. Only
                  meaningful when running in SageMaker mode.
              logs (bool): Whether to show the logs produced by the job.
                  Only meaningful when wait is True.
          """
          # use self.model_id, self._s3_model_output_path as the model to evaluate
          # Model object has already been initialized with up-to-date DDb record.
          model_artifact_path = self.model_record.get_model_artifact_path()
          rl_estimator_args = self._get_rl_estimator_args(eval=True)
          rl_estimator_args["model_channel_name"] = "pretrained_model"
          rl_estimator_args["model_uri"] = model_artifact_path
  
          if manifest_file_path:
              inputs = sagemaker.inputs.TrainingInput(
                  s3_data=manifest_file_path, s3_data_type="ManifestFile"
              )
              if local_mode:
                  rl_estimator_args["hyperparameters"].update({"local_mode_manifest": True})
  
          else:
              inputs = input_data_s3_prefix
  
          # (dict[str, str] or dict[str, sagemaker.session.s3_input]) for evaluation channel
          eval_channel_inputs = {EVAL_CHANNEL: inputs}
          self.rl_estimator = RLEstimator(**rl_estimator_args)
  
          # update to save eval_data_s3_path in DDb as well, or
          # update to read from SM describe call... maybe will not work in local mode but.
          eval_data_s3_path = (
              manifest_file_path if (manifest_file_path is not None) else input_data_s3_prefix
          )
  
          # we keep eval job state as pending, before the SM job has been submitted.
          # the syncer function should update this state, based on SM job status.
          self.model_record.add_new_evaluation_job_info(
              evaluation_job_name=evaluation_job_name, eval_data_s3_path=eval_data_s3_path
          )
          self.model_db_client.update_model_record(self._jsonify())
  
          # The following local variables (unsaved to DDb) make evaluation job non-resumable.
          self.log_output = None
          self.local_mode = local_mode
  
          if local_mode:
              # Capture eval score by regex expression
              # log should contain only one "average loss = some number" pattern
              with CaptureStdout() as log_output:
                  self.rl_estimator.fit(
                      job_name=evaluation_job_name, inputs=eval_channel_inputs, wait=wait, logs=logs
                  )
  
              self.log_output = "\n".join(log_output)
              logger.debug(self.log_output)
          else:
              self.rl_estimator.fit(
                  job_name=evaluation_job_name, inputs=eval_channel_inputs, wait=wait, logs=logs
              )
code_tokens: def evaluate ( self , input_data_s3_prefix = None , manifest_file_path = None , evaluation_job_name = None , local_mode = True , wait = False , logs = True , ) : model_artifact_path = self . model_record . get_model_artifact_path ( ) rl_estimator_args = self . _get_rl_estimator_args ( eval = True ) rl_estimator_args [ "model_channel_name" ] = "pretrained_model" rl_estimator_args [ "model_uri" ] = model_artifact_path if manifest_file_path : inputs = sagemaker . inputs . TrainingInput ( s3_data = manifest_file_path , s3_data_type = "ManifestFile" ) if local_mode : rl_estimator_args [ "hyperparameters" ] . update ( { "local_mode_manifest" : True } ) else : inputs = input_data_s3_prefix eval_channel_inputs = { EVAL_CHANNEL : inputs } self . rl_estimator = RLEstimator ( ** rl_estimator_args ) eval_data_s3_path = ( manifest_file_path if ( manifest_file_path is not None ) else input_data_s3_prefix ) self . model_record . add_new_evaluation_job_info ( evaluation_job_name = evaluation_job_name , eval_data_s3_path = eval_data_s3_path ) self . model_db_client . update_model_record ( self . _jsonify ( ) ) self . log_output = None self . local_mode = local_mode if local_mode : with CaptureStdout ( ) as log_output : self . rl_estimator . fit ( job_name = evaluation_job_name , inputs = eval_channel_inputs , wait = wait , logs = logs ) self . log_output = "\n" . join ( log_output ) logger . debug ( self . log_output ) else : self . rl_estimator . fit ( job_name = evaluation_job_name , inputs = eval_channel_inputs , wait = wait , logs = logs )
short_docstring: A Estimator fit() call to start a model evaluation job.
short_docstring_tokens: A Estimator fit () call to start a model evaluation job .
comment: """A Estimator fit() call to start a model evaluation job.
  
          Args:
              input_data_s3_prefix (str): Defines the location of s3 data used for evaluation
              manifest_file_path (str): Manifest file used to provide evaluation data.
              evaluation_job_name (str): Unique Sagemaker job name to identify the evaluation job
              local_mode (bool): Whether the evaluation job is running on local mode
              wait (bool): Whether the call should wait until the job completes. Only
                  meaningful when running in SageMaker mode.
              logs (bool): Whether to show the logs produced by the job.
                  Only meaningful when wait is True.
          """$SEP$# use self.model_id, self._s3_model_output_path as the model to evaluate$SEP$# Model object has already been initialized with up-to-date DDb record.$SEP$# (dict[str, str] or dict[str, sagemaker.session.s3_input]) for evaluation channel$SEP$# update to save eval_data_s3_path in DDb as well, or$SEP$# update to read from SM describe call... maybe will not work in local mode but.$SEP$# we keep eval job state as pending, before the SM job has been submitted.$SEP$# the syncer function should update this state, based on SM job status.$SEP$# The following local variables (unsaved to DDb) make evaluation job non-resumable.$SEP$# Capture eval score by regex expression$SEP$# log should contain only one "average loss = some number" pattern