def _parse_pre_yarn_history_log(lines):
 result = {}
 task_to_counters = {}
 for record in _parse_pre_yarn_history_records(lines):
  fields = record['fields']
  if ((record['type'] == 'Job') and ('COUNTERS' in fields)):
   result['counters'] = _parse_pre_yarn_counters(fields['COUNTERS'])
  elif ((record['type'] == 'Task') and ('COUNTERS' in fields) and ('TASKID' in fields)):
   task_id = fields['TASKID']
   counters = _parse_pre_yarn_counters(fields['COUNTERS'])
   task_to_counters[task_id] = counters
  elif ((record['type'] in ('MapAttempt', 'ReduceAttempt')) and ('TASK_ATTEMPT_ID' in fields) and (fields.get('TASK_STATUS') == 'FAILED') and fields.get('ERROR')):
   result.setdefault('errors', [])
   result['errors'].append(dict(hadoop_error=dict(message=fields['ERROR'], start_line=record['start_line'], num_lines=record['num_lines']), attempt_id=fields['TASK_ATTEMPT_ID']))
 if (('counters' not in result) and task_to_counters):
  result['counters'] = _sum_counters(*task_to_counters.values())
 return result

def _parse_pre_yarn_history_log ( lines ) : result = { } task_to_counters = { } for record in _parse_pre_yarn_history_records ( lines ) : fields = record [ 'fields' ] if ( ( record [ 'type' ] == 'Job' ) and ( 'COUNTERS' in fields ) ) : result [ 'counters' ] = _parse_pre_yarn_counters ( fields [ 'COUNTERS' ] ) elif ( ( record [ 'type' ] == 'Task' ) and ( 'COUNTERS' in fields ) and ( 'TASKID' in fields ) ) : task_id = fields [ 'TASKID' ] counters = _parse_pre_yarn_counters ( fields [ 'COUNTERS' ] ) task_to_counters [ task_id ] = counters elif ( ( record [ 'type' ] in ( 'MapAttempt' , 'ReduceAttempt' ) ) and ( 'TASK_ATTEMPT_ID' in fields ) and ( fields . get ( 'TASK_STATUS' ) == 'FAILED' ) and fields . get ( 'ERROR' ) ) : result . setdefault ( 'errors' , [ ] ) result [ 'errors' ] . append ( dict ( hadoop_error = dict ( message = fields [ 'ERROR' ] , start_line = record [ 'start_line' ] , num_lines = record [ 'num_lines' ] ) , attempt_id = fields [ 'TASK_ATTEMPT_ID' ] ) ) if ( ( 'counters' not in result ) and task_to_counters ) : result [ 'counters' ] = _sum_counters ( * task_to_counters . values ( ) ) return result
collect useful info from a pre-yarn history file .