Paginate a BigQuery result set

  # Use a page token to loop through a page of results at a time when 
 # querying a BigQuery dataset larger than the Workflows memory limit 
 # This workflow queries a public dataset to determine the most common 
 # names in the US between 1910 and 2013 
 main 
 : 
  
 params 
 : 
  
 [ 
 input 
 ] 
  
 steps 
 : 
  
 - 
  
 init 
 : 
  
 assign 
 : 
  
 - 
  
 pageToken 
 : 
  
 null 
  
 - 
  
 startQuery 
 : 
  
 call 
 : 
  
 googleapis.bigquery.v2.jobs.insert 
  
 args 
 : 
  
 projectId 
 : 
  
 ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")} 
  
 body 
 : 
  
 configuration 
 : 
  
 query 
 : 
  
 useLegacySql 
 : 
  
 false 
  
 # Remove LIMIT from the query to iterate through all results 
  
 query 
 : 
  
 SELECT name, SUM(number) AS total FROM `bigquery-public-data.usa_names.usa_1910_2013` GROUP BY name ORDER BY total DESC LIMIT 50 
  
 result 
 : 
  
 query 
  
 - 
  
 getPage 
 : 
  
 call 
 : 
  
 googleapis.bigquery.v2.jobs.getQueryResults 
  
 args 
 : 
  
 projectId 
 : 
  
 ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")} 
  
 jobId 
 : 
  
 ${query.jobReference.jobId} 
  
 maxResults 
 : 
  
 10 
  
 pageToken 
 : 
  
 ${pageToken} 
  
 result 
 : 
  
 page 
  
 - 
  
 processPage 
 : 
  
 for 
 : 
  
 value 
 : 
  
 row 
  
 in 
 : 
  
 ${page.rows} 
  
 steps 
 : 
  
 - 
  
 processRow 
 : 
  
 call 
 : 
  
 sys.log 
  
 args 
 : 
  
 data 
 : 
  
 ${row} 
  
 - 
  
 checkIfDone 
 : 
  
 switch 
 : 
  
 - 
  
 condition 
 : 
  
 ${"pageToken" in page and page.pageToken != ""} 
  
 assign 
 : 
  
 - 
  
 pageToken 
 : 
  
 ${page.pageToken} 
  
 next 
 : 
  
 getPage