original_docstring: 
      Import data file (JSON or CSV) from Google Storage into bigquery table.
      
docstring: Import data file (JSON or CSV) from Google Storage into bigquery table.
docstring_tokens: Import data file ( JSON or CSV ) from Google Storage into bigquery table .
code: def load_data_to_table(dataset_id, table_id, gsfn, schema, wait=True, verbose=False, maxbad=None, 
                         format=None, skiprows=None,
                         project_id=DEFAULT_PROJECT_ID
                         ):
      '''
      Import data file (JSON or CSV) from Google Storage into bigquery table.
      '''
  
      project_ref = dict(projectId=project_id)
      table_ref = dict(datasetId=dataset_id, projectId=project_id, tableId=table_id)
  
      config = {'load': {'sourceUris': [gsfn],
                         'schema': {'fields': schema},
                         "destinationTable": table_ref,
                         'sourceFormat': "NEWLINE_DELIMITED_JSON",
                         # "maxBadRecords": 0,
                         'writeDisposition': 'WRITE_TRUNCATE',
                         }
                }
                
      if skiprows is not None:
          config['load']["skipLeadingRows"] = skiprows
  
      if format=='csv':
          config['load']['sourceFormat'] = 'CSV'
  
      if maxbad is not None:
          config['load']['maxBadRecords'] = maxbad
  
      job_id = 'load_%s_%d' % (table_id, time.time())
      job_ref = {'jobId': job_id,
                 'projectId': project_id}
      
      job = {'jobReference': job_ref, 'configuration': config}
  
      print "[bqutil] loading table %s from %s, running job %s" % (table_id, gsfn, job_id)
      sys.stdout.flush()
  
      if verbose:
          print job
  
      for k in range(10):
          try:
              jobret = jobs.insert(body=job, **project_ref).execute()
              break
          except Exception as err:
              print "[bqutil] oops!  Failed to insert job=%s" % job
              if (k==9):
                  raise
              if 'HttpError 500' in str(err):
                  print err
                  print "--> 500 error, retrying in 30 sec"
                  time.sleep(30)
                  continue
              raise
  
      job = jobret
  
      if verbose:
          print "job=", json.dumps(job, indent=4)
        
      if verbose:
          job_list = jobs.list( stateFilter=['pending', 'running'], **project_ref).execute()
          print "job list: ", job_list
  
      if not wait:
          return
  
      nerr = 0
      while job['status']['state'] <> 'DONE':
          try:
              job = jobs.get(**job_ref).execute()
          except Exception as err:
              if "Internal Error" in str(err):
                  nerr += 1
                  if nerr > 10:
                      raise
                  time.sleep(10)
                  continue
  
      status = job['status']
      print "[bqutil] job status: ", status
  
      if 'errors' in status:
          print "[bqutil] ERROR!  ", status['errors']
          print "job = ", json.dumps(job, indent=4)
          raise Exception('BQ Error creating table')
      else:
          me = getpass.getuser()
          project_name = get_project_name(project_id)
          txt = "Data loaded from %s by %s / bqutil on %s\n" % (gsfn, me, datetime.datetime.now())
          txt += 'see job: https://bigquery.cloud.google.com/results/%s:%s\n' % (project_name, job_id)
          txt += 'see table: https://bigquery.cloud.google.com/table/%s:%s.%s\n\n' % (project_name, dataset_id, table_id)
          add_description_to_table(dataset_id, table_id, txt, project_id=project_id)
  
      return job
code_tokens: def load_data_to_table ( dataset_id , table_id , gsfn , schema , wait = True , verbose = False , maxbad = None , format = None , skiprows = None , project_id = DEFAULT_PROJECT_ID ) : project_ref = dict ( projectId = project_id ) table_ref = dict ( datasetId = dataset_id , projectId = project_id , tableId = table_id ) config = { 'load' : { 'sourceUris' : [ gsfn ] , 'schema' : { 'fields' : schema } , "destinationTable" : table_ref , 'sourceFormat' : "NEWLINE_DELIMITED_JSON" , 'writeDisposition' : 'WRITE_TRUNCATE' , } } if skiprows is not None : config [ 'load' ] [ "skipLeadingRows" ] = skiprows if format == 'csv' : config [ 'load' ] [ 'sourceFormat' ] = 'CSV' if maxbad is not None : config [ 'load' ] [ 'maxBadRecords' ] = maxbad job_id = 'load_%s_%d' % ( table_id , time . time ( ) ) job_ref = { 'jobId' : job_id , 'projectId' : project_id } job = { 'jobReference' : job_ref , 'configuration' : config } print "[bqutil] loading table %s from %s, running job %s" % ( table_id , gsfn , job_id ) sys . stdout . flush ( ) if verbose : print job for k in range ( 10 ) : try : jobret = jobs . insert ( body = job , ** project_ref ) . execute ( ) break except Exception as err : print "[bqutil] oops!  Failed to insert job=%s" % job if ( k == 9 ) : raise if 'HttpError 500' in str ( err ) : print err print "--> 500 error, retrying in 30 sec" time . sleep ( 30 ) continue raise job = jobret if verbose : print "job=" , json . dumps ( job , indent = 4 ) if verbose : job_list = jobs . list ( stateFilter = [ 'pending' , 'running' ] , ** project_ref ) . execute ( ) print "job list: " , job_list if not wait : return nerr = 0 while job [ 'status' ] [ 'state' ] <> 'DONE' : try : job = jobs . get ( ** job_ref ) . execute ( ) except Exception as err : if "Internal Error" in str ( err ) : nerr += 1 if nerr > 10 : raise time . sleep ( 10 ) continue status = job [ 'status' ] print "[bqutil] job status: " , status if 'errors' in status : print "[bqutil] ERROR!  " , status [ 'errors' ] print "job = " , json . dumps ( job , indent = 4 ) raise Exception ( 'BQ Error creating table' ) else : me = getpass . getuser ( ) project_name = get_project_name ( project_id ) txt = "Data loaded from %s by %s / bqutil on %s\n" % ( gsfn , me , datetime . datetime . now ( ) ) txt += 'see job: https://bigquery.cloud.google.com/results/%s:%s\n' % ( project_name , job_id ) txt += 'see table: https://bigquery.cloud.google.com/table/%s:%s.%s\n\n' % ( project_name , dataset_id , table_id ) add_description_to_table ( dataset_id , table_id , txt , project_id = project_id ) return job
short_docstring: Import data file (JSON or CSV) from Google Storage into bigquery table.
short_docstring_tokens: Import data file ( JSON or CSV ) from Google Storage into bigquery table .
comment: '''
      Import data file (JSON or CSV) from Google Storage into bigquery table.
      '''$SEP$# "maxBadRecords": 0,