original_docstring: 
      Run SQL query to create a new table.
      
docstring: Run SQL query to create a new table.
docstring_tokens: Run SQL query to create a new table .
code: def create_bq_table(dataset_id, table_id, sql, verbose=False, overwrite=False, wait=True, 
                      logger=default_logger, project_id=DEFAULT_PROJECT_ID,
                      allowLargeResults=False,
                      output_project_id=DEFAULT_PROJECT_ID):
      '''
      Run SQL query to create a new table.
      '''
  
      if not sql:
          raise Exception('[bqutil] Cannot create_bq_table: sql=%s' % sql)
  
      project_ref = dict(projectId=project_id)
      table_ref = dict(datasetId=dataset_id, projectId=output_project_id, tableId=table_id)
  
      if overwrite in ["append", 'APPEND']:
          wd = "WRITE_APPEND"
      elif overwrite==True:
          wd = "WRITE_TRUNCATE"
      else:
          wd = "WRITE_EMPTY"	
  
      config = {'query': { 'query': sql,
                           'destinationTable': table_ref,
                           'writeDisposition': wd,
                           'allowLargeResults': allowLargeResults,
                           }
                }
                
      job_id = 'create_%s_%s_%d' % (dataset_id, table_id, time.time())
      job_ref = {'jobId': job_id,
                 'projectId': project_id}
      
      job = {'jobReference': job_ref, 'configuration': config}
  
      logger("[bqutil] Creating table %s, running job %s" % (table_id, job_id))
      sys.stdout.flush()
  
      if verbose:
          print job
  
      for k in range(10):
          try:
              jobret = jobs.insert(body=job, **project_ref).execute()
              break
          except Exception as err:
              print "[bqutil] oops!  Failed to insert job=%s" % job
              if (k==9):
                  raise
              if 'HttpError 500' in str(err):
                  print err
                  print "--> 500 error, retrying in 30 sec"
                  time.sleep(30)
                  continue
              raise
  
      if verbose:
          print "job=", json.dumps(job, indent=4)
        
      if verbose:
          job_list = jobs.list( stateFilter=['pending', 'running'], **project_ref).execute()
          print "job list: ", job_list
  
      if not wait:
          return
  
      ecnt = 0
      while job.get('status', {}).get('state', None) <> 'DONE':
          if 'status' not in job:
              ecnt += 1
              if (ecnt > 2):
                  logger("[bqutil] Error!  no job status?  job ret = %s" % job)
              if (ecnt > 5):
                  raise Exception('BQ Error getting job status')
          else:
              ecnt = 0
          try:
              job = jobs.get(**job_ref).execute()
          except Exception as err:
              print "[bqutil] oops!  Failed to execute jobs.get=%s" % (job_ref)
  
      status = job['status']
      logger( "[bqutil] job status: %s" % status )
  
      if 'errors' in status:
          logger( "[bqutil] ERROR!  %s" % str(status['errors']) )
          logger( "job = %s" % json.dumps(job, indent=4))
          raise Exception('BQ Error creating table')
  
      elif status['state']=='DONE':
  
          nbytes = int(job['statistics']['query']['totalBytesProcessed'])
          logger( "[bqutil] Total bytes processed (proportional to $$$ cost): %10.2f kB" % (nbytes/1024.0) )
  
          ctime = int(job['statistics']['creationTime'])
          etime = int(job['statistics']['endTime'])
          dt = (etime - ctime)/1000.0
          logger( "[bqutil] Job run time: %8.2f seconds" % dt)
  
          # Patch the table to add a description
          if not wd=='WRITE_APPEND':
              try:
                  me = getpass.getuser()
              except Exception as err:
                  me = "gae"
              txt = 'Computed by %s / bqutil at %s processing %s bytes in %8.2f sec\nwith this SQL: %s' % (me, datetime.datetime.now(), 
                                                                                                           nbytes,
                                                                                                           dt,
                                                                                                           sql)
              project_name = get_project_name(project_id)
              output_project_name = get_project_name(output_project_id)
      
              txt += '\n'
              txt += 'see job: https://bigquery.cloud.google.com/results/%s:%s\n' % (project_name, job_id)
              txt += 'see table: https://bigquery.cloud.google.com/table/%s:%s.%s\n' % (output_project_name, dataset_id, table_id)
              logger(txt)
      
              add_description_to_table(dataset_id, table_id, txt, project_id=output_project_id)
  
      return job
code_tokens: def create_bq_table ( dataset_id , table_id , sql , verbose = False , overwrite = False , wait = True , logger = default_logger , project_id = DEFAULT_PROJECT_ID , allowLargeResults = False , output_project_id = DEFAULT_PROJECT_ID ) : if not sql : raise Exception ( '[bqutil] Cannot create_bq_table: sql=%s' % sql ) project_ref = dict ( projectId = project_id ) table_ref = dict ( datasetId = dataset_id , projectId = output_project_id , tableId = table_id ) if overwrite in [ "append" , 'APPEND' ] : wd = "WRITE_APPEND" elif overwrite == True : wd = "WRITE_TRUNCATE" else : wd = "WRITE_EMPTY" config = { 'query' : { 'query' : sql , 'destinationTable' : table_ref , 'writeDisposition' : wd , 'allowLargeResults' : allowLargeResults , } } job_id = 'create_%s_%s_%d' % ( dataset_id , table_id , time . time ( ) ) job_ref = { 'jobId' : job_id , 'projectId' : project_id } job = { 'jobReference' : job_ref , 'configuration' : config } logger ( "[bqutil] Creating table %s, running job %s" % ( table_id , job_id ) ) sys . stdout . flush ( ) if verbose : print job for k in range ( 10 ) : try : jobret = jobs . insert ( body = job , ** project_ref ) . execute ( ) break except Exception as err : print "[bqutil] oops!  Failed to insert job=%s" % job if ( k == 9 ) : raise if 'HttpError 500' in str ( err ) : print err print "--> 500 error, retrying in 30 sec" time . sleep ( 30 ) continue raise if verbose : print "job=" , json . dumps ( job , indent = 4 ) if verbose : job_list = jobs . list ( stateFilter = [ 'pending' , 'running' ] , ** project_ref ) . execute ( ) print "job list: " , job_list if not wait : return ecnt = 0 while job . get ( 'status' , { } ) . get ( 'state' , None ) <> 'DONE' : if 'status' not in job : ecnt += 1 if ( ecnt > 2 ) : logger ( "[bqutil] Error!  no job status?  job ret = %s" % job ) if ( ecnt > 5 ) : raise Exception ( 'BQ Error getting job status' ) else : ecnt = 0 try : job = jobs . get ( ** job_ref ) . execute ( ) except Exception as err : print "[bqutil] oops!  Failed to execute jobs.get=%s" % ( job_ref ) status = job [ 'status' ] logger ( "[bqutil] job status: %s" % status ) if 'errors' in status : logger ( "[bqutil] ERROR!  %s" % str ( status [ 'errors' ] ) ) logger ( "job = %s" % json . dumps ( job , indent = 4 ) ) raise Exception ( 'BQ Error creating table' ) elif status [ 'state' ] == 'DONE' : nbytes = int ( job [ 'statistics' ] [ 'query' ] [ 'totalBytesProcessed' ] ) logger ( "[bqutil] Total bytes processed (proportional to $$$ cost): %10.2f kB" % ( nbytes / 1024.0 ) ) ctime = int ( job [ 'statistics' ] [ 'creationTime' ] ) etime = int ( job [ 'statistics' ] [ 'endTime' ] ) dt = ( etime - ctime ) / 1000.0 logger ( "[bqutil] Job run time: %8.2f seconds" % dt ) if not wd == 'WRITE_APPEND' : try : me = getpass . getuser ( ) except Exception as err : me = "gae" txt = 'Computed by %s / bqutil at %s processing %s bytes in %8.2f sec\nwith this SQL: %s' % ( me , datetime . datetime . now ( ) , nbytes , dt , sql ) project_name = get_project_name ( project_id ) output_project_name = get_project_name ( output_project_id ) txt += '\n' txt += 'see job: https://bigquery.cloud.google.com/results/%s:%s\n' % ( project_name , job_id ) txt += 'see table: https://bigquery.cloud.google.com/table/%s:%s.%s\n' % ( output_project_name , dataset_id , table_id ) logger ( txt ) add_description_to_table ( dataset_id , table_id , txt , project_id = output_project_id ) return job
short_docstring: Run SQL query to create a new table.
short_docstring_tokens: Run SQL query to create a new table .
comment: '''
      Run SQL query to create a new table.
      '''$SEP$# Patch the table to add a description