func (c *Client) largeDataPagedQuery(service *bigquery.Service, pageSize int, dataset, project, queryStr string, dataChan chan Data) ([][]interface{}, []string, error) {
	c.printDebug("largeDataPagedQuery starting")
	ts := time.Now()
	// start query
	tableRef := bigquery.TableReference{DatasetId: dataset, ProjectId: project, TableId: c.tempTableName}
	jobConfigQuery := bigquery.JobConfigurationQuery{}

	datasetRef := &bigquery.DatasetReference{
		DatasetId: dataset,
		ProjectId: project,
	}

	jobConfigQuery.AllowLargeResults = true
	jobConfigQuery.Query = queryStr
	jobConfigQuery.DestinationTable = &tableRef
	jobConfigQuery.DefaultDataset = datasetRef
	if !c.flattenResults {
		c.printDebug("setting FlattenResults to false")
		// need a pointer to bool
		f := false
		jobConfigQuery.FlattenResults = &f
	}
	jobConfigQuery.WriteDisposition = "WRITE_TRUNCATE"
	jobConfigQuery.CreateDisposition = "CREATE_IF_NEEDED"

	jobConfig := bigquery.JobConfiguration{}

	jobConfig.Query = &jobConfigQuery

	job := bigquery.Job{}
	job.Configuration = &jobConfig

	jobInsert := service.Jobs.Insert(project, &job)
	runningJob, jerr := jobInsert.Do()

	if jerr != nil {
		c.printDebug("Error inserting job!", jerr)
		if dataChan != nil {
			dataChan <- Data{Err: jerr}
		}
		return nil, nil, jerr
	}

	var qr *bigquery.GetQueryResultsResponse
	var rows [][]interface{}
	var headers []string
	var err error

	// Periodically, job references are not created, but errors are also not thrown.
	// In this scenario, retry up to 5 times to get a job reference before giving up.
	for i := 1; ; i++ {
		r := service.Jobs.GetQueryResults(project, runningJob.JobReference.JobId)
		r.TimeoutMs(c.RequestTimeout)
		qr, err = r.Do()

		headers, rows = c.headersAndRows(qr.Schema, qr.Rows)

		if i >= maxRequestRetry || qr.JobReference != nil || err != nil {
			if i > 1 {
				c.printDebug("Took %v tries to get a job reference", i)
			}
			break
		}
	}

	if err == nil && qr.JobReference == nil {
		err = fmt.Errorf("missing job reference")
	}

	if err != nil {
		c.printDebug("Error loading query: ", err)
		if dataChan != nil {
			dataChan <- Data{Err: err}
		}
		return nil, nil, err
	}

	rows, headers, err = c.processPagedQuery(qr.JobReference, qr.PageToken, dataChan, headers, rows)
	c.printDebug("largeDataPagedQuery completed in ", time.Now().Sub(ts).Seconds(), "s")

	return rows, headers, err
}
func (c *Client) largeDataPagedQuery(service *bigquery.Service, pageSize int, dataset, project, queryStr string, dataChan chan Data) ([][]interface{}, []string, error) {
	c.printDebug("largeDataPagedQuery starting")
	ts := time.Now()
	// start query
	tableRef := bigquery.TableReference{DatasetId: dataset, ProjectId: project, TableId: c.tempTableName}
	jobConfigQuery := bigquery.JobConfigurationQuery{}

	datasetRef := &bigquery.DatasetReference{
		DatasetId: dataset,
		ProjectId: project,
	}

	jobConfigQuery.AllowLargeResults = true
	jobConfigQuery.Query = queryStr
	jobConfigQuery.DestinationTable = &tableRef
	jobConfigQuery.DefaultDataset = datasetRef
	if !c.flattenResults {
		c.printDebug("setting FlattenResults to false")
		// need a pointer to bool
		f := false
		jobConfigQuery.FlattenResults = &f
	}
	jobConfigQuery.WriteDisposition = "WRITE_TRUNCATE"
	jobConfigQuery.CreateDisposition = "CREATE_IF_NEEDED"

	jobConfig := bigquery.JobConfiguration{}

	jobConfig.Query = &jobConfigQuery

	job := bigquery.Job{}
	job.Configuration = &jobConfig

	jobInsert := service.Jobs.Insert(project, &job)
	runningJob, jerr := jobInsert.Do()

	if jerr != nil {
		c.printDebug("Error inserting job!", jerr)
		if dataChan != nil {
			dataChan <- Data{Err: jerr}
		}
		return nil, nil, jerr
	}

	var qr *bigquery.GetQueryResultsResponse
	var rows [][]interface{}
	var headers []string
	var err error

	// Periodically, job references are not created, but errors are also not thrown.
	// In this scenario, retry up to 5 times to get a job reference before giving up.
	for i := 1; ; i++ {
		r := service.Jobs.GetQueryResults(project, runningJob.JobReference.JobId)
		r.TimeoutMs(c.RequestTimeout)
		qr, err = r.Do()

		headers, rows = c.headersAndRows(qr.Schema, qr.Rows)

		if i >= maxRequestRetry || qr.JobReference != nil || err != nil {
			if i > 1 {
				c.printDebug("Took %v tries to get a job reference", i)
			}
			break
		}
	}

	if err == nil && qr.JobReference == nil {
		err = fmt.Errorf("missing job reference")
	}

	if err != nil {
		c.printDebug("Error loading query: ", err)
		if dataChan != nil {
			dataChan <- Data{Err: err}
		}
		return nil, nil, err
	}

	rows, headers, err = c.processPagedQuery(qr.JobReference, qr.PageToken, dataChan, headers, rows)
	c.printDebug("largeDataPagedQuery completed in ", time.Now().Sub(ts).Seconds(), "s")

	return rows, headers, err
}
func ( c * Client ) largeDataPagedQuery ( service * bigquery . Service , pageSize int , dataset , project , queryStr string , dataChan chan Data ) ( [ ] [ ] interface { } , [ ] string , error ) { c . printDebug ( " " ) ts := time . Now ( ) // start query tableRef := bigquery . TableReference { DatasetId : dataset , ProjectId : project , TableId : c . tempTableName } jobConfigQuery := bigquery . JobConfigurationQuery { } datasetRef := & bigquery . DatasetReference { DatasetId : dataset , ProjectId : project , } jobConfigQuery . AllowLargeResults = true jobConfigQuery . Query = queryStr jobConfigQuery . DestinationTable = & tableRef jobConfigQuery . DefaultDataset = datasetRef if ! c . flattenResults { c . printDebug ( " " ) // need a pointer to bool f := false jobConfigQuery . FlattenResults = & f } jobConfigQuery . WriteDisposition = " " jobConfigQuery . CreateDisposition = " " jobConfig := bigquery . JobConfiguration { } jobConfig . Query = & jobConfigQuery job := bigquery . Job { } job . Configuration = & jobConfig jobInsert := service . Jobs . Insert ( project , & job ) runningJob , jerr := jobInsert . Do ( ) if jerr != nil { c . printDebug ( " " , jerr ) if dataChan != nil { dataChan <- Data { Err : jerr } } return nil , nil , jerr } var qr * bigquery . GetQueryResultsResponse var rows [ ] [ ] interface { } var headers [ ] string var err error // Periodically, job references are not created, but errors are also not thrown. // In this scenario, retry up to 5 times to get a job reference before giving up. for i := 1 ; ; i ++ { r := service . Jobs . GetQueryResults ( project , runningJob . JobReference . JobId ) r . TimeoutMs ( c . RequestTimeout ) qr , err = r . Do ( ) headers , rows = c . headersAndRows ( qr . Schema , qr . Rows ) if i >= maxRequestRetry || qr . JobReference != nil || err != nil { if i > 1 { c . printDebug ( " " , i ) } break } } if err == nil && qr . JobReference == nil { err = fmt . Errorf ( " " ) } if err != nil { c . printDebug ( " " , err ) if dataChan != nil { dataChan <- Data { Err : err } } return nil , nil , err } rows , headers , err = c . processPagedQuery ( qr . JobReference , qr . PageToken , dataChan , headers , rows ) c . printDebug ( " " , time . Now ( ) . Sub ( ts ) . Seconds ( ) , " " ) return rows , headers , err }
// largeDataPagedQuery builds a job and inserts it into the job queue allowing the flexibility to set the custom AllowLargeResults flag for the job
largeDataPagedQuery builds a job and inserts it into the job queue allowing the flexibility to set the custom AllowLargeResults flag for the job