Troubleshoot service accounts in GKE

 #!/bin/bash 
 # Set your project ID 
 project_id 
 = 
  PROJECT_ID 
 
 project_number 
 = 
 $( 
gcloud  
projects  
describe  
 " 
 $project_id 
 " 
  
--format = 
 "value(projectNumber)" 
 ) 
 declare 
  
-a  
all_service_accounts declare 
  
-a  
sa_missing_permissions # Function to check if a service account has a specific permission 
 # $1: project_id 
 # $2: service_account 
 # $3: permission 
service_account_has_permission () 
  
 { 
  
 local 
  
 project_id 
 = 
 " 
 $1 
 " 
  
 local 
  
 service_account 
 = 
 " 
 $2 
 " 
  
 local 
  
 permission 
 = 
 " 
 $3 
 " 
  
 local 
  
 roles 
 = 
 $( 
gcloud  
projects  
get-iam-policy  
 " 
 $project_id 
 " 
  
 \ 
  
--flatten = 
 "bindings[].members" 
  
 \ 
  
--format = 
 "table[no-heading](bindings.role)" 
  
 \ 
  
--filter = 
 "bindings.members:\" 
 $service_account 
 \"" 
 ) 
  
 for 
  
role  
 in 
  
 $roles 
 ; 
  
 do 
  
 if 
  
role_has_permission  
 " 
 $role 
 " 
  
 " 
 $permission 
 " 
 ; 
  
 then 
  
 echo 
  
 "Yes" 
  
 # Has permission 
  
 return 
  
 fi 
  
 done 
  
 echo 
  
 "No" 
  
 # Does not have permission 
 } 
 # Function to check if a role has the specific permission 
 # $1: role 
 # $2: permission 
role_has_permission () 
  
 { 
  
 local 
  
 role 
 = 
 " 
 $1 
 " 
  
 local 
  
 permission 
 = 
 " 
 $2 
 " 
  
gcloud  
iam  
roles  
describe  
 " 
 $role 
 " 
  
--format = 
 "json" 
  
 | 
  
 \ 
  
jq  
-r  
 ".includedPermissions" 
  
 | 
  
 \ 
  
grep  
-q  
 " 
 $permission 
 " 
 } 
 # Function to add $1 into the service account array all_service_accounts 
 # $1: service account 
add_service_account () 
  
 { 
  
 local 
  
 service_account 
 = 
 " 
 $1 
 " 
  
 all_service_accounts 
 +=( 
  
 ${ 
 service_account 
 } 
  
 ) 
 } 
 # Function to add service accounts into the global array all_service_accounts for a Standard GKE cluster 
 # $1: project_id 
 # $2: location 
 # $3: cluster_name 
add_service_accounts_for_standard () 
  
 { 
  
 local 
  
 project_id 
 = 
 " 
 $1 
 " 
  
 local 
  
 cluster_location 
 = 
 " 
 $2 
 " 
  
 local 
  
 cluster_name 
 = 
 " 
 $3 
 " 
  
 while 
  
 read 
  
nodepool ; 
  
 do 
  
 nodepool_name 
 = 
 $( 
 echo 
  
 " 
 $nodepool 
 " 
  
 | 
  
awk  
 '{print $1}' 
 ) 
  
 if 
  
 [[ 
  
 " 
 $nodepool_name 
 " 
  
 == 
  
 "" 
  
 ]] 
 ; 
  
 then 
  
 # skip the empty line which is from running `gcloud container node-pools list` in GCP console 
  
 continue 
  
 fi 
  
 while 
  
 read 
  
nodepool_details ; 
  
 do 
  
 service_account 
 = 
 $( 
 echo 
  
 " 
 $nodepool_details 
 " 
  
 | 
  
awk  
 '{print $1}' 
 ) 
  
 if 
  
 [[ 
  
 " 
 $service_account 
 " 
  
 == 
  
 "default" 
  
 ]] 
 ; 
  
 then 
  
 service_account 
 = 
 " 
 ${ 
 project_number 
 } 
 -compute@developer.gserviceaccount.com" 
  
 fi 
  
 if 
  
 [[ 
  
-n  
 " 
 $service_account 
 " 
  
 ]] 
 ; 
  
 then 
  
 printf 
  
 "%-60s| %-40s| %-40s| %-10s| %-20s\n" 
  
 $service_account 
  
 $project_id 
  
 $cluster_name 
  
 $cluster_location 
  
 $nodepool_name 
  
add_service_account  
 " 
 ${ 
 service_account 
 } 
 " 
  
 else 
  
 echo 
  
 "cannot find service account for node pool 
 $project_id 
 \t 
 $cluster_name 
 \t 
 $cluster_location 
 \t 
 $nodepool_details 
 " 
  
 fi 
  
 done 
  
 <<< 
  
 " 
 $( 
gcloud  
container  
node-pools  
describe  
 " 
 $nodepool_name 
 " 
  
--cluster  
 " 
 $cluster_name 
 " 
  
--zone  
 " 
 $cluster_location 
 " 
  
--project  
 " 
 $project_id 
 " 
  
--format = 
 "table[no-heading](config.serviceAccount)" 
 ) 
 " 
  
 done 
  
 <<< 
  
 " 
 $( 
gcloud  
container  
node-pools  
list  
--cluster  
 " 
 $cluster_name 
 " 
  
--zone  
 " 
 $cluster_location 
 " 
  
--project  
 " 
 $project_id 
 " 
  
--format = 
 "table[no-heading](name)" 
 ) 
 " 
 } 
 # Function to add service accounts into the global array all_service_accounts for an Autopilot GKE cluster 
 # Autopilot cluster only has one node service account. 
 # $1: project_id 
 # $2: location 
 # $3: cluster_name 
add_service_account_for_autopilot (){ 
  
 local 
  
 project_id 
 = 
 " 
 $1 
 " 
  
 local 
  
 cluster_location 
 = 
 " 
 $2 
 " 
  
 local 
  
 cluster_name 
 = 
 " 
 $3 
 " 
  
 while 
  
 read 
  
service_account ; 
  
 do 
  
 if 
  
 [[ 
  
 " 
 $service_account 
 " 
  
 == 
  
 "default" 
  
 ]] 
 ; 
  
 then 
  
 service_account 
 = 
 " 
 ${ 
 project_number 
 } 
 -compute@developer.gserviceaccount.com" 
  
 fi 
  
 if 
  
 [[ 
  
-n  
 " 
 $service_account 
 " 
  
 ]] 
 ; 
  
 then 
  
 printf 
  
 "%-60s| %-40s| %-40s| %-10s| %-20s\n" 
  
 $service_account 
  
 $project_id 
  
 $cluster_name 
  
 $cluster_location 
  
 $nodepool_name 
  
add_service_account  
 " 
 ${ 
 service_account 
 } 
 " 
  
 else 
  
 echo 
  
 "cannot find service account" 
  
 for 
  
cluster  
 " 
 $project_id 
 \t 
 $cluster_name 
 \t 
 $cluster_location 
 \t" 
  
 fi 
  
 done 
  
 <<< 
  
 " 
 $( 
gcloud  
container  
clusters  
describe  
 " 
 $cluster_name 
 " 
  
--location  
 " 
 $cluster_location 
 " 
  
--project  
 " 
 $project_id 
 " 
  
--format = 
 "table[no-heading](autoscaling.autoprovisioningNodePoolDefaults.serviceAccount)" 
 ) 
 " 
 } 
 # Function to check whether the cluster is an Autopilot cluster or not 
 # $1: project_id 
 # $2: location 
 # $3: cluster_name 
is_autopilot_cluster () 
  
 { 
  
 local 
  
 project_id 
 = 
 " 
 $1 
 " 
  
 local 
  
 cluster_location 
 = 
 " 
 $2 
 " 
  
 local 
  
 cluster_name 
 = 
 " 
 $3 
 " 
  
 autopilot 
 = 
 $( 
gcloud  
container  
clusters  
describe  
 " 
 $cluster_name 
 " 
  
--location  
 " 
 $cluster_location 
 " 
  
--format = 
 "table[no-heading](autopilot.enabled)" 
 ) 
  
 echo 
  
 " 
 $autopilot 
 " 
 } 
 echo 
  
 "--- 1. List all service accounts in all GKE node pools" 
 printf 
  
 "%-60s| %-40s| %-40s| %-10s| %-20s\n" 
  
 "service_account" 
  
 "project_id" 
  
 "cluster_name" 
  
 "cluster_location" 
  
 "nodepool_name" 
 while 
  
 read 
  
cluster ; 
  
 do 
  
 cluster_name 
 = 
 $( 
 echo 
  
 " 
 $cluster 
 " 
  
 | 
  
awk  
 '{print $1}' 
 ) 
  
 cluster_location 
 = 
 $( 
 echo 
  
 " 
 $cluster 
 " 
  
 | 
  
awk  
 '{print $2}' 
 ) 
  
 # how to find a cluster is a Standard cluster or an Autopilot cluster 
  
 autopilot 
 = 
 $( 
is_autopilot_cluster  
 " 
 $project_id 
 " 
  
 " 
 $cluster_location 
 " 
  
 " 
 $cluster_name 
 " 
 ) 
  
 if 
  
 [[ 
  
 " 
 $autopilot 
 " 
  
 == 
  
 "True" 
  
 ]] 
 ; 
  
 then 
  
add_service_account_for_autopilot  
 " 
 $project_id 
 " 
  
 " 
 $cluster_location 
 " 
  
 " 
 $cluster_name 
 " 
  
 else 
  
add_service_accounts_for_standard  
 " 
 $project_id 
 " 
  
 " 
 $cluster_location 
 " 
  
 " 
 $cluster_name 
 " 
  
 fi 
 done 
  
 <<< 
  
 " 
 $( 
gcloud  
container  
clusters  
list  
--project  
 " 
 $project_id 
 " 
  
--format = 
 "value(name,location)" 
 ) 
 " 
 echo 
  
 "--- 2. Check if service accounts have permissions" 
 unique_service_accounts 
 =( 
 $( 
 echo 
  
 " 
 ${ 
 all_service_accounts 
 [@] 
 } 
 " 
  
 | 
  
tr  
 ' ' 
  
 '\n' 
  
 | 
  
sort  
-u  
 | 
  
tr  
 '\n' 
  
 ' ' 
 ) 
 ) 
 echo 
  
 "Service accounts: 
 ${ 
 unique_service_accounts 
 [@] 
 } 
 " 
 printf 
  
 "%-60s| %-40s| %-40s| %-20s\n" 
  
 "service_account" 
  
 "has_logging_permission" 
  
 "has_monitoring_permission" 
  
 "has_performance_hpa_metric_write_permission" 
 for 
  
sa  
 in 
  
 " 
 ${ 
 unique_service_accounts 
 [@] 
 } 
 " 
 ; 
  
 do 
  
 logging_permission 
 = 
 $( 
service_account_has_permission  
 " 
 $project_id 
 " 
  
 " 
 $sa 
 " 
  
 "logging.logEntries.create" 
 ) 
  
 time_series_create_permission 
 = 
 $( 
service_account_has_permission  
 " 
 $project_id 
 " 
  
 " 
 $sa 
 " 
  
 "monitoring.timeSeries.create" 
 ) 
  
 metric_descriptors_create_permission 
 = 
 $( 
service_account_has_permission  
 " 
 $project_id 
 " 
  
 " 
 $sa 
 " 
  
 "monitoring.metricDescriptors.create" 
 ) 
  
 if 
  
 [[ 
  
 " 
 $time_series_create_permission 
 " 
  
 == 
  
 "No" 
  
 || 
  
 " 
 $metric_descriptors_create_permission 
 " 
  
 == 
  
 "No" 
  
 ]] 
 ; 
  
 then 
  
 monitoring_permission 
 = 
 "No" 
  
 else 
  
 monitoring_permission 
 = 
 "Yes" 
  
 fi 
  
 performance_hpa_metric_write_permission 
 = 
 $( 
service_account_has_permission  
 " 
 $project_id 
 " 
  
 " 
 $sa 
 " 
  
 "autoscaling.sites.writeMetrics" 
 ) 
  
 printf 
  
 "%-60s| %-40s| %-40s| %-20s\n" 
  
 $sa 
  
 $logging_permission 
  
 $monitoring_permission 
  
 $performance_hpa_metric_write_permission 
  
 if 
  
 [[ 
  
 " 
 $logging_permission 
 " 
  
 == 
  
 "No" 
  
 || 
  
 " 
 $monitoring_permission 
 " 
  
 == 
  
 "No" 
  
 || 
  
 " 
 $performance_hpa_metric_write_permission 
 " 
  
 == 
  
 "No" 
  
 ]] 
 ; 
  
 then 
  
 sa_missing_permissions 
 +=( 
  
 ${ 
 sa 
 } 
  
 ) 
  
 fi 
 done 
 echo 
  
 "--- 3. List all service accounts that don't have the above permissions" 
 if 
  
 [[ 
  
 " 
 ${# 
 sa_missing_permissions[@]}" 
  
-gt  
 0 
  
 ]] 
 ; 
  
 then 
  
 printf 
  
 "Grant roles/container.defaultNodeServiceAccount to the following service accounts: %s\n" 
  
 " 
 ${ 
 sa_missing_permissions 
 [@] 
 } 
 " 
 else 
  
 echo 
  
 "All service accounts have the above permissions" 
 fi