def _interpret_spark_task_logs(fs, matches, partial=True, log_callback=None):
 result = {}
 for match in matches:
  error = {}
  stderr_path = match['path']
  if log_callback:
   log_callback(stderr_path)
  stderr_error = _parse_task_syslog(_cat_log(fs, stderr_path))
  if stderr_error.get('hadoop_error'):
   stderr_error['hadoop_error']['path'] = stderr_path
   error.update(stderr_error)
  else:
   continue
  stdout_path = (match.get('stdout') or {}).get('path')
  check_stdout = error.pop('check_stdout', None)
  if (stdout_path and check_stdout):
   if log_callback:
    log_callback(stdout_path)
   task_error = _parse_task_stderr(_cat_log(fs, stdout_path))
   if task_error:
    task_error['path'] = stdout_path
    error['task_error'] = task_error
  for id_key in ('attempt_id', 'container_id'):
   if (id_key in match):
    error[id_key] = match[id_key]
  _add_implied_task_id(error)
  result.setdefault('errors', [])
  result['errors'].append(error)
  if partial:
   result['partial'] = True
   break
 return result

def _interpret_spark_task_logs ( fs , matches , partial = True , log_callback = None ) : result = { } for match in matches : error = { } stderr_path = match [ 'path' ] if log_callback : log_callback ( stderr_path ) stderr_error = _parse_task_syslog ( _cat_log ( fs , stderr_path ) ) if stderr_error . get ( 'hadoop_error' ) : stderr_error [ 'hadoop_error' ] [ 'path' ] = stderr_path error . update ( stderr_error ) else : continue stdout_path = ( match . get ( 'stdout' ) or { } ) . get ( 'path' ) check_stdout = error . pop ( 'check_stdout' , None ) if ( stdout_path and check_stdout ) : if log_callback : log_callback ( stdout_path ) task_error = _parse_task_stderr ( _cat_log ( fs , stdout_path ) ) if task_error : task_error [ 'path' ] = stdout_path error [ 'task_error' ] = task_error for id_key in ( 'attempt_id' , 'container_id' ) : if ( id_key in match ) : error [ id_key ] = match [ id_key ] _add_implied_task_id ( error ) result . setdefault ( 'errors' , [ ] ) result [ 'errors' ] . append ( error ) if partial : result [ 'partial' ] = True break return result
look for errors in spark task stderr .