airflow/contrib/hooks/sagemaker_hook.py
def describe_training_job_with_log(self, job_name, positions, stream_names,
                                       instance_count, state, last_description,
                                       last_describe_job_call):
        """
        Return the training job info associated with job_name and print CloudWatch logs
        """
        log_group = '/aws/sagemaker/TrainingJobs'

        if len(stream_names) < instance_count:
            # Log streams are created whenever a container starts writing to stdout/err, so this list
            # may be dynamic until we have a stream for every instance.
            logs_conn = self.get_log_conn()
            try:
                streams = logs_conn.describe_log_streams(
                    logGroupName=log_group,
                    logStreamNamePrefix=job_name + '/',
                    orderBy='LogStreamName',
                    limit=instance_count
                )
                stream_names = [s['logStreamName'] for s in streams['logStreams']]
                positions.update([(s, Position(timestamp=0, skip=0))
                                  for s in stream_names if s not in positions])
            except logs_conn.exceptions.ResourceNotFoundException:
                # On the very first training job run on an account, there's no log group until
                # the container starts logging, so ignore any errors thrown about that
                pass

        if len(stream_names) > 0:
            for idx, event in self.multi_stream_iter(log_group, stream_names, positions):
                self.log.info(event['message'])
                ts, count = positions[stream_names[idx]]
                if event['timestamp'] == ts:
                    positions[stream_names[idx]] = Position(timestamp=ts, skip=count + 1)
                else:
                    positions[stream_names[idx]] = Position(timestamp=event['timestamp'], skip=1)

        if state == LogState.COMPLETE:
            return state, last_description, last_describe_job_call

        if state == LogState.JOB_COMPLETE:
            state = LogState.COMPLETE
        elif time.time() - last_describe_job_call >= 30:
            description = self.describe_training_job(job_name)
            last_describe_job_call = time.time()

            if secondary_training_status_changed(description, last_description):
                self.log.info(secondary_training_status_message(description, last_description))
                last_description = description

            status = description['TrainingJobStatus']

            if status not in self.non_terminal_states:
                state = LogState.JOB_COMPLETE
        return state, last_description, last_describe_job_call
def describe_training_job_with_log(self, job_name, positions, stream_names,
                                       instance_count, state, last_description,
                                       last_describe_job_call):
        log_group = '/aws/sagemaker/TrainingJobs'

        if len(stream_names) < instance_count:
            # Log streams are created whenever a container starts writing to stdout/err, so this list
            # may be dynamic until we have a stream for every instance.
            logs_conn = self.get_log_conn()
            try:
                streams = logs_conn.describe_log_streams(
                    logGroupName=log_group,
                    logStreamNamePrefix=job_name + '/',
                    orderBy='LogStreamName',
                    limit=instance_count
                )
                stream_names = [s['logStreamName'] for s in streams['logStreams']]
                positions.update([(s, Position(timestamp=0, skip=0))
                                  for s in stream_names if s not in positions])
            except logs_conn.exceptions.ResourceNotFoundException:
                # On the very first training job run on an account, there's no log group until
                # the container starts logging, so ignore any errors thrown about that
                pass

        if len(stream_names) > 0:
            for idx, event in self.multi_stream_iter(log_group, stream_names, positions):
                self.log.info(event['message'])
                ts, count = positions[stream_names[idx]]
                if event['timestamp'] == ts:
                    positions[stream_names[idx]] = Position(timestamp=ts, skip=count + 1)
                else:
                    positions[stream_names[idx]] = Position(timestamp=event['timestamp'], skip=1)

        if state == LogState.COMPLETE:
            return state, last_description, last_describe_job_call

        if state == LogState.JOB_COMPLETE:
            state = LogState.COMPLETE
        elif time.time() - last_describe_job_call >= 30:
            description = self.describe_training_job(job_name)
            last_describe_job_call = time.time()

            if secondary_training_status_changed(description, last_description):
                self.log.info(secondary_training_status_message(description, last_description))
                last_description = description

            status = description['TrainingJobStatus']

            if status not in self.non_terminal_states:
                state = LogState.JOB_COMPLETE
        return state, last_description, last_describe_job_call
def describe_training_job_with_log ( self , job_name , positions , stream_names , instance_count , state , last_description , last_describe_job_call ) : log_group = '/aws/sagemaker/TrainingJobs' if len ( stream_names ) < instance_count : # Log streams are created whenever a container starts writing to stdout/err, so this list # may be dynamic until we have a stream for every instance. logs_conn = self . get_log_conn ( ) try : streams = logs_conn . describe_log_streams ( logGroupName = log_group , logStreamNamePrefix = job_name + '/' , orderBy = 'LogStreamName' , limit = instance_count ) stream_names = [ s [ 'logStreamName' ] for s in streams [ 'logStreams' ] ] positions . update ( [ ( s , Position ( timestamp = 0 , skip = 0 ) ) for s in stream_names if s not in positions ] ) except logs_conn . exceptions . ResourceNotFoundException : # On the very first training job run on an account, there's no log group until # the container starts logging, so ignore any errors thrown about that pass if len ( stream_names ) > 0 : for idx , event in self . multi_stream_iter ( log_group , stream_names , positions ) : self . log . info ( event [ 'message' ] ) ts , count = positions [ stream_names [ idx ] ] if event [ 'timestamp' ] == ts : positions [ stream_names [ idx ] ] = Position ( timestamp = ts , skip = count + 1 ) else : positions [ stream_names [ idx ] ] = Position ( timestamp = event [ 'timestamp' ] , skip = 1 ) if state == LogState . COMPLETE : return state , last_description , last_describe_job_call if state == LogState . JOB_COMPLETE : state = LogState . COMPLETE elif time . time ( ) - last_describe_job_call >= 30 : description = self . describe_training_job ( job_name ) last_describe_job_call = time . time ( ) if secondary_training_status_changed ( description , last_description ) : self . log . info ( secondary_training_status_message ( description , last_description ) ) last_description = description status = description [ 'TrainingJobStatus' ] if status not in self . non_terminal_states : state = LogState . JOB_COMPLETE return state , last_description , last_describe_job_call
Return the training job info associated with job_name and print CloudWatch logs
Return the training job info associated with job_name and print CloudWatch logs