Using speech-to-speech translation

 # Set language_code to your desired language, in this case, Mandarin Chinese. 
 speech_config 
 = 
 SpeechConfig 
 ( 
 language_code 
 = 
 "cmn" 
 ) 
 config 
 = 
 LiveConnectConfig 
 ( 
 response_modalities 
 = 
 [ 
 "AUDIO" 
 ], 
 speech_config 
 = 
 speech_config 
 , 
 input_audio_transcription 
 = 
 input_transcription 
 , 
 output_audio_transcription 
 = 
 output_transcription 
 , 
 ) 
 audio_file 
 = 
 Part 
 . 
 from_uri 
 ( 
 file_uri 
 = 
 audio_url 
 , 
 mime_type 
 = 
 "audio/mpeg" 
 ) 
 contents 
 = 
 [ 
 audio_file 
 ] 
 response 
 = 
 client 
 . 
 models 
 . 
 generate_content 
 ( 
 model 
 = 
 MODEL_ID 
 , 
 contents 
 = 
 contents 
 ) 
 display 
 ( 
 Markdown 
 ( 
 response 
 . 
 text 
 )) 

 import 
  
 asyncio 
 # Set model generation_config 
 CONFIG 
 = 
 { 
 "response_modalities" 
 : 
 [ 
 "AUDIO" 
 ], 
 "speech_config" 
 : 
 { 
 "language_code" 
 : 
 "cmn" 
 , 
 }, 
 } 
 headers 
 = 
 { 
 "Content-Type" 
 : 
 "application/json" 
 , 
 "Authorization" 
 : 
 f 
 "Bearer 
 { 
 bearer_token 
 [ 
 0 
 ] 
 } 
 " 
 , 
 } 
 # Connect to the server 
 async 
 with 
 connect 
 ( 
 SERVICE_URL 
 , 
 additional_headers 
 = 
 headers 
 ) 
 as 
 ws 
 : 
 # Setup the session 
 await 
 ws 
 . 
 send 
 ( 
 json 
 . 
 dumps 
 ( 
 { 
 "setup" 
 : 
 { 
 "model" 
 : 
 MODEL 
 , 
 "generation_config" 
 : 
 CONFIG 
 , 
 "input_audio_transcription" 
 : 
 {}, 
 "output_audio_transcription" 
 : 
 {}, 
 "enable_speech_to_speech_translation" 
 : 
 True 
 , 
 } 
 } 
 ) 
 ) 
 # Receive setup response 
 raw_response 
 = 
 await 
 ws 
 . 
 recv 
 ( 
 decode 
 = 
 False 
 ) 
 setup_response 
 = 
 json 
 . 
 loads 
 ( 
 raw_response 
 . 
 decode 
 ( 
 "ascii" 
 )) 
 print 
 ( 
 setup_response 
 ) 
 msg 
 = 
 { 
 "realtime_input" 
 : 
 { 
 "audio" 
 : 
 { 
 "mime_type" 
 : 
 "audio/pcm" 
 , 
 "data" 
 : 
 base64 
 . 
 b64encode 
 ( 
 wav_data 
 ) 
 . 
 decode 
 ( 
 'utf-8' 
 ), 
 } 
 } 
 } 
 await 
 ws 
 . 
 send 
 ( 
 json 
 . 
 dumps 
 ( 
 msg 
 )) 
 overall_responses 
 = 
 [] 
 timeout_seconds 
 = 
 10 
 # Set timeout to 3 seconds 
 # Receive chucks of server response with a timeout 
 try 
 : 
 while 
 True 
 : 
 try 
 : 
 raw_response 
 = 
 await 
 asyncio 
 . 
 wait_for 
 ( 
 ws 
 . 
 recv 
 ( 
 decode 
 = 
 False 
 ), 
 timeout_seconds 
 ) 
 response 
 = 
 json 
 . 
 loads 
 ( 
 raw_response 
 . 
 decode 
 ()) 
 server_content 
 = 
 response 
 . 
 pop 
 ( 
 "serverContent" 
 , 
 None 
 ) 
 if 
 server_content 
 is 
 None 
 : 
 break 
 # Input Transcription. 
 input_transcription 
 = 
 server_content 
 . 
 pop 
 ( 
 "inputTranscription" 
 , 
 None 
 ) 
 if 
 input_transcription 
 is 
 not 
 None 
 : 
 raw_text 
 = 
 input_transcription 
 . 
 pop 
 ( 
 "text" 
 , 
 None 
 ) 
 if 
 raw_text 
 is 
 not 
 None 
 : 
 display 
 ( 
 Markdown 
 ( 
 f 
 "**Input >** 
 { 
 raw_text 
 } 
 " 
 )) 
 # Output Transcription. 
 output_transcription 
 = 
 server_content 
 . 
 pop 
 ( 
 "outputTranscription" 
 , 
 None 
 ) 
 if 
 output_transcription 
 is 
 not 
 None 
 : 
 raw_text 
 = 
 output_transcription 
 . 
 pop 
 ( 
 "text" 
 , 
 None 
 ) 
 if 
 raw_text 
 is 
 not 
 None 
 : 
 display 
 ( 
 Markdown 
 ( 
 f 
 "**Response >** 
 { 
 raw_text 
 } 
 " 
 )) 
 model_turn 
 = 
 server_content 
 . 
 pop 
 ( 
 "modelTurn" 
 , 
 None 
 ) 
 if 
 model_turn 
 is 
 not 
 None 
 : 
 parts 
 = 
 model_turn 
 . 
 pop 
 ( 
 "parts" 
 , 
 None 
 ) 
 if 
 parts 
 is 
 not 
 None 
 : 
 for 
 part 
 in 
 parts 
 : 
 pcm_data 
 = 
 base64 
 . 
 b64decode 
 ( 
 part 
 [ 
 "inlineData" 
 ][ 
 "data" 
 ]) 
 overall_responses 
 . 
 append 
 ( 
 np 
 . 
 frombuffer 
 ( 
 pcm_data 
 , 
 dtype 
 = 
 np 
 . 
 int16 
 )) 
 # End of turn 
 # turn_complete = server_content.pop("turnComplete", None) 
 # if turn_complete: 
 #     break 
 except 
 asyncio 
 . 
 TimeoutError 
 : 
 print 
 ( 
 f 
 "Timeout: No response received from the websocket within 
 { 
 timeout_seconds 
 } 
 seconds." 
 ) 
 if 
 overall_responses 
 : 
 display 
 ( 
 Audio 
 ( 
 np 
 . 
 concatenate 
 ( 
 overall_responses 
 ), 
 rate 
 = 
 24000 
 , 
 autoplay 
 = 
 True 
 )) 
 break 
 # Exit the loop on timeout 
 except 
 websockets 
 . 
 exceptions 
 . 
 ConnectionClosed 
 as 
 e 
 : 
 print 
 ( 
 f 
 "Connection closed by exception, code: 
 { 
 e 
 . 
 code 
 } 
 , reason: 
 { 
 e 
 . 
 reason 
 } 
 " 
 ) 
 if 
 overall_responses 
 : 
 display 
 ( 
 Audio 
 ( 
 np 
 . 
 concatenate 
 ( 
 overall_responses 
 ), 
 rate 
 = 
 24000 
 , 
 autoplay 
 = 
 True 
 )) 
 break 
 # Exit the loop on connection closed 
 except 
 Exception 
 as 
 e 
 : 
 print 
 ( 
 f 
 "An unexpected error occurred: 
 { 
 e 
 } 
 " 
 ) 
 if 
 overall_responses 
 : 
 display 
 ( 
 Audio 
 ( 
 np 
 . 
 concatenate 
 ( 
 overall_responses 
 ), 
 rate 
 = 
 24000 
 , 
 autoplay 
 = 
 True 
 )) 
 break 
 # Exit the loop on other exceptions 
 finally 
 : 
 try 
 : 
 await 
 ws 
 . 
 close 
 ( 
 code 
 = 
 1000 
 , 
 reason 
 = 
 "Normal closure" 
 ) 
 #example close 
 except 
 websockets 
 . 
 exceptions 
 . 
 ConnectionClosed 
 as 
 e 
 : 
 print 
 ( 
 f 
 "Connection closed by exception, code: 
 { 
 e 
 . 
 code 
 } 
 , reason: 
 { 
 e 
 . 
 reason 
 } 
 " 
 ) 
 except 
 Exception 
 as 
 e 
 : 
 print 
 ( 
 f 
 "An unexpected error occurred: 
 { 
 e 
 } 
 " 
 )