Load data into a column-based time partitioning table

  import 
  
 ( 
  
 "context" 
  
 "fmt" 
  
 "time" 
  
 "cloud.google.com/go/bigquery" 
 ) 
 // importPartitionedTable demonstrates specifing time partitioning for a BigQuery table when loading 
 // CSV data from Cloud Storage. 
 func 
  
 importPartitionedTable 
 ( 
 projectID 
 , 
  
 destDatasetID 
 , 
  
 destTableID 
  
 string 
 ) 
  
 error 
  
 { 
  
 // projectID := "my-project-id" 
  
 // datasetID := "mydataset" 
  
 // tableID := "mytable" 
  
 ctx 
  
 := 
  
 context 
 . 
 Background 
 () 
  
 client 
 , 
  
 err 
  
 := 
  
 bigquery 
 . 
 NewClient 
 ( 
 ctx 
 , 
  
 projectID 
 ) 
  
 if 
  
 err 
  
 != 
  
 nil 
  
 { 
  
 return 
  
 fmt 
 . 
 Errorf 
 ( 
 "bigquery.NewClient: %w" 
 , 
  
 err 
 ) 
  
 } 
  
 defer 
  
 client 
 . 
 Close 
 () 
  
 gcsRef 
  
 := 
  
 bigquery 
 . 
  NewGCSReference 
 
 ( 
 "gs://cloud-samples-data/bigquery/us-states/us-states-by-date.csv" 
 ) 
  
 gcsRef 
 . 
 SkipLeadingRows 
  
 = 
  
 1 
  
 gcsRef 
 . 
  Schema 
 
  
 = 
  
 bigquery 
 . 
  Schema 
 
 { 
  
 { 
 Name 
 : 
  
 "name" 
 , 
  
 Type 
 : 
  
 bigquery 
 . 
  StringFieldType 
 
 }, 
  
 { 
 Name 
 : 
  
 "post_abbr" 
 , 
  
 Type 
 : 
  
 bigquery 
 . 
  StringFieldType 
 
 }, 
  
 { 
 Name 
 : 
  
 "date" 
 , 
  
 Type 
 : 
  
 bigquery 
 . 
  DateFieldType 
 
 }, 
  
 } 
  
 loader 
  
 := 
  
 client 
 . 
 Dataset 
 ( 
 destDatasetID 
 ). 
 Table 
 ( 
 destTableID 
 ). 
  LoaderFrom 
 
 ( 
 gcsRef 
 ) 
  
 loader 
 . 
  TimePartitioning 
 
  
 = 
  
& bigquery 
 . 
  TimePartitioning 
 
 { 
  
 Field 
 : 
  
 "date" 
 , 
  
 Expiration 
 : 
  
 90 
  
 * 
  
 24 
  
 * 
  
 time 
 . 
 Hour 
 , 
  
 } 
  
 loader 
 . 
 WriteDisposition 
  
 = 
  
 bigquery 
 . 
  WriteEmpty 
 
  
 job 
 , 
  
 err 
  
 := 
  
 loader 
 . 
 Run 
 ( 
 ctx 
 ) 
  
 if 
  
 err 
  
 != 
  
 nil 
  
 { 
  
 return 
  
 err 
  
 } 
  
 status 
 , 
  
 err 
  
 := 
  
 job 
 . 
 Wait 
 ( 
 ctx 
 ) 
  
 if 
  
 err 
  
 != 
  
 nil 
  
 { 
  
 return 
  
 err 
  
 } 
  
 if 
  
 status 
 . 
  Err 
 
 () 
  
 != 
  
 nil 
  
 { 
  
 return 
  
 fmt 
 . 
 Errorf 
 ( 
 "job completed with error: %w" 
 , 
  
 status 
 . 
  Err 
 
 ()) 
  
 } 
  
 return 
  
 nil 
 } 
 

  import 
  
 com.google.cloud.bigquery. BigQuery 
 
 ; 
 import 
  
 com.google.cloud.bigquery. BigQueryException 
 
 ; 
 import 
  
 com.google.cloud.bigquery. BigQueryOptions 
 
 ; 
 import 
  
 com.google.cloud.bigquery. Field 
 
 ; 
 import 
  
 com.google.cloud.bigquery. FormatOptions 
 
 ; 
 import 
  
 com.google.cloud.bigquery. Job 
 
 ; 
 import 
  
 com.google.cloud.bigquery. JobId 
 
 ; 
 import 
  
 com.google.cloud.bigquery. JobInfo 
 
 ; 
 import 
  
 com.google.cloud.bigquery. LoadJobConfiguration 
 
 ; 
 import 
  
 com.google.cloud.bigquery. Schema 
 
 ; 
 import 
  
 com.google.cloud.bigquery. StandardSQLTypeName 
 
 ; 
 import 
  
 com.google.cloud.bigquery. TableId 
 
 ; 
 import 
  
 com.google.cloud.bigquery. TimePartitioning 
 
 ; 
 import 
  
 java. time 
.Duration 
 ; 
 import 
  
 java. time 
.temporal.ChronoUnit 
 ; 
 import 
  
 java.util.UUID 
 ; 
 public 
  
 class 
 LoadPartitionedTable 
  
 { 
  
 public 
  
 static 
  
 void 
  
 main 
 ( 
 String 
 [] 
  
 args 
 ) 
  
 throws 
  
 Exception 
  
 { 
  
 // TODO(developer): Replace these variables before running the sample. 
  
 String 
  
 datasetName 
  
 = 
  
 "MY_DATASET_NAME" 
 ; 
  
 String 
  
 tableName 
  
 = 
  
 "MY_TABLE_NAME" 
 ; 
  
 String 
  
 sourceUri 
  
 = 
  
 "/path/to/file.csv" 
 ; 
  
 loadPartitionedTable 
 ( 
 datasetName 
 , 
  
 tableName 
 , 
  
 sourceUri 
 ); 
  
 } 
  
 public 
  
 static 
  
 void 
  
 loadPartitionedTable 
 ( 
 String 
  
 datasetName 
 , 
  
 String 
  
 tableName 
 , 
  
 String 
  
 sourceUri 
 ) 
  
 throws 
  
 Exception 
  
 { 
  
 try 
  
 { 
  
 // Initialize client that will be used to send requests. This client only needs to be created 
  
 // once, and can be reused for multiple requests. 
  
  BigQuery 
 
  
 bigquery 
  
 = 
  
  BigQueryOptions 
 
 . 
 getDefaultInstance 
 (). 
 getService 
 (); 
  
  TableId 
 
  
 tableId 
  
 = 
  
  TableId 
 
 . 
 of 
 ( 
 datasetName 
 , 
  
 tableName 
 ); 
  
  Schema 
 
  
 schema 
  
 = 
  
  Schema 
 
 . 
 of 
 ( 
  
  Field 
 
 . 
 of 
 ( 
 "name" 
 , 
  
  StandardSQLTypeName 
 
 . 
 STRING 
 ), 
  
  Field 
 
 . 
 of 
 ( 
 "post_abbr" 
 , 
  
  StandardSQLTypeName 
 
 . 
 STRING 
 ), 
  
  Field 
 
 . 
 of 
 ( 
 "date" 
 , 
  
  StandardSQLTypeName 
 
 . 
 DATE 
 )); 
  
 // Configure time partitioning. For full list of options, see: 
  
 // https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TimePartitioning 
  
  TimePartitioning 
 
  
 partitioning 
  
 = 
  
  TimePartitioning 
 
 . 
 newBuilder 
 ( 
  TimePartitioning 
 
 . 
 Type 
 . 
 DAY 
 ) 
  
 . 
 setField 
 ( 
 "date" 
 ) 
  
 . 
  setExpirationMs 
 
 ( 
 Duration 
 . 
 of 
 ( 
 90 
 , 
  
 ChronoUnit 
 . 
 DAYS 
 ). 
 toMillis 
 ()) 
  
 . 
 build 
 (); 
  
  LoadJobConfiguration 
 
  
 loadJobConfig 
  
 = 
  
  LoadJobConfiguration 
 
 . 
 builder 
 ( 
 tableId 
 , 
  
 sourceUri 
 ) 
  
 . 
 setFormatOptions 
 ( 
  FormatOptions 
 
 . 
  csv 
 
 ()) 
  
 . 
 setSchema 
 ( 
 schema 
 ) 
  
 . 
 setTimePartitioning 
 ( 
 partitioning 
 ) 
  
 . 
 build 
 (); 
  
 // Create a job ID so that we can safely retry. 
  
  JobId 
 
  
 jobId 
  
 = 
  
  JobId 
 
 . 
 of 
 ( 
 UUID 
 . 
 randomUUID 
 (). 
 toString 
 ()); 
  
  Job 
 
  
 loadJob 
  
 = 
  
 bigquery 
 . 
  create 
 
 ( 
 JobInfo 
 . 
 newBuilder 
 ( 
 loadJobConfig 
 ). 
 setJobId 
 ( 
 jobId 
 ). 
 build 
 ()); 
  
 // Load data from a GCS parquet file into the table 
  
 // Blocks until this load table job completes its execution, either failing or succeeding. 
  
  Job 
 
  
 completedJob 
  
 = 
  
 loadJob 
 . 
  waitFor 
 
 (); 
  
 // Check for errors 
  
 if 
  
 ( 
 completedJob 
  
 == 
  
 null 
 ) 
  
 { 
  
 throw 
  
 new 
  
 Exception 
 ( 
 "Job not executed since it no longer exists." 
 ); 
  
 } 
  
 else 
  
 if 
  
 ( 
 completedJob 
 . 
 getStatus 
 (). 
 getError 
 () 
  
 != 
  
 null 
 ) 
  
 { 
  
 // You can also look at queryJob.getStatus().getExecutionErrors() for all 
  
 // errors, not just the latest one. 
  
 throw 
  
 new 
  
 Exception 
 ( 
  
 "BigQuery was unable to load into the table due to an error: \n" 
  
 + 
  
 loadJob 
 . 
 getStatus 
 (). 
 getError 
 ()); 
  
 } 
  
 System 
 . 
 out 
 . 
 println 
 ( 
 "Data successfully loaded into time partitioned table during load job" 
 ); 
  
 } 
  
 catch 
  
 ( 
  BigQueryException 
 
  
 | 
  
 InterruptedException 
  
 e 
 ) 
  
 { 
  
 System 
 . 
 out 
 . 
 println 
 ( 
  
 "Data not loaded into time partitioned table during load job \n" 
  
 + 
  
 e 
 . 
 toString 
 ()); 
  
 } 
  
 } 
 } 
 

  // Import the Google Cloud client libraries 
 const 
  
 { 
 BigQuery 
 } 
  
 = 
  
 require 
 ( 
 ' @google-cloud/bigquery 
' 
 ); 
 const 
  
 { 
 Storage 
 } 
  
 = 
  
 require 
 ( 
 ' @google-cloud/storage 
' 
 ); 
 // Instantiate clients 
 const 
  
 bigquery 
  
 = 
  
 new 
  
  BigQuery 
 
 (); 
 const 
  
 storage 
  
 = 
  
 new 
  
 Storage 
 (); 
 /** 
 * This sample loads the CSV file at 
 * https://storage.googleapis.com/cloud-samples-data/bigquery/us-states/us-states.csv 
 * 
 * TODO(developer): Replace the following lines with the path to your file. 
 */ 
 const 
  
 bucketName 
  
 = 
  
 'cloud-samples-data' 
 ; 
 const 
  
 filename 
  
 = 
  
 'bigquery/us-states/us-states-by-date.csv' 
 ; 
 async 
  
 function 
  
 loadTablePartitioned 
 () 
  
 { 
  
 // Load data into a table that uses column-based time partitioning. 
  
 /** 
 * TODO(developer): Uncomment the following lines before running the sample. 
 */ 
  
 // const datasetId = 'my_dataset'; 
  
 // const tableId = 'my_new_table'; 
  
 // Configure the load job. For full list of options, see: 
  
 // https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad 
  
 const 
  
 partitionConfig 
  
 = 
  
 { 
  
 type 
 : 
  
 'DAY' 
 , 
  
 expirationMs 
 : 
  
 '7776000000' 
 , 
  
 // 90 days 
  
 field 
 : 
  
 ' date 
' 
 , 
  
 }; 
  
 const 
  
 metadata 
  
 = 
  
 { 
  
 sourceFormat 
 : 
  
 'CSV' 
 , 
  
 skipLeadingRows 
 : 
  
 1 
 , 
  
 schema 
 : 
  
 { 
  
 fields 
 : 
  
 [ 
  
 { 
 name 
 : 
  
 'name' 
 , 
  
 type 
 : 
  
 'STRING' 
 }, 
  
 { 
 name 
 : 
  
 'post_abbr' 
 , 
  
 type 
 : 
  
 'STRING' 
 }, 
  
 { 
 name 
 : 
  
 ' date 
' 
 , 
  
 type 
 : 
  
 'DATE' 
 }, 
  
 ], 
  
 }, 
  
 location 
 : 
  
 'US' 
 , 
  
 timePartitioning 
 : 
  
 partitionConfig 
 , 
  
 }; 
  
 // Load data from a Google Cloud Storage file into the table 
  
 const 
  
 [ 
 job 
 ] 
  
 = 
  
 await 
  
 bigquery 
  
 . 
 dataset 
 ( 
 datasetId 
 ) 
  
 . 
 table 
 ( 
 tableId 
 ) 
  
 . 
  load 
 
 ( 
 storage 
 . 
 bucket 
 ( 
 bucketName 
 ). 
 file 
 ( 
 filename 
 ), 
  
 metadata 
 ); 
  
 // load() waits for the job to finish 
  
 console 
 . 
 log 
 ( 
 `Job 
 ${ 
  job 
 
 . 
 id 
 } 
 completed.` 
 ); 
 } 
 

  from 
  
 google.cloud 
  
 import 
  bigquery 
 
 # Construct a BigQuery client object. 
 client 
 = 
  bigquery 
 
 . 
  Client 
 
 () 
 # TODO(developer): Set table_id to the ID of the table to create. 
 # table_id = "your-project.your_dataset.your_table_name" 
 job_config 
 = 
  bigquery 
 
 . 
  LoadJobConfig 
 
 ( 
 schema 
 = 
 [ 
  bigquery 
 
 . 
  SchemaField 
 
 ( 
 "name" 
 , 
 "STRING" 
 ), 
  bigquery 
 
 . 
  SchemaField 
 
 ( 
 "post_abbr" 
 , 
 "STRING" 
 ), 
  bigquery 
 
 . 
  SchemaField 
 
 ( 
 "date" 
 , 
 "DATE" 
 ), 
 ], 
 skip_leading_rows 
 = 
 1 
 , 
 time_partitioning 
 = 
  bigquery 
 
 . 
  TimePartitioning 
 
 ( 
 type_ 
 = 
  bigquery 
 
 . 
  TimePartitioningType 
 
 . 
  DAY 
 
 , 
 field 
 = 
 "date" 
 , 
 # Name of the column to use for partitioning. 
 expiration_ms 
 = 
 7776000000 
 , 
 # 90 days. 
 ), 
 ) 
 uri 
 = 
 "gs://cloud-samples-data/bigquery/us-states/us-states-by-date.csv" 
 load_job 
 = 
 client 
 . 
  load_table_from_uri 
 
 ( 
 uri 
 , 
 table_id 
 , 
 job_config 
 = 
 job_config 
 ) 
 # Make an API request. 
 load_job 
 . 
 result 
 () 
 # Wait for the job to complete. 
 table 
 = 
 client 
 . 
  get_table 
 
 ( 
 table_id 
 ) 
 print 
 ( 
 "Loaded 
 {} 
 rows to table 
 {} 
 " 
 . 
 format 
 ( 
 table 
 . 
  num_rows 
 
 , 
 table_id 
 ))