This sample demonstrates how to inspect a Cloud Storage resource and create de-identified copies of the files.
Explore further
For detailed documentation that includes this code sample, see the following:
Code sample
C#
To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries .
To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment .
using
Google.Api.Gax.ResourceNames
;
using
Google.Cloud.Dlp.V2
;
using
System.Linq
;
public
class
DeidentifyDataStoredInCloudStorage
{
public
static
DlpJob
Deidentify
(
string
projectId
,
string
gcsInputPath
,
string
unstructuredDeidentifyTemplatePath
,
string
structuredDeidentifyTemplatePath
,
string
imageRedactionTemplatePath
,
string
gcsOutputPath
,
string
datasetId
,
string
tableId
)
{
// Instantiate the client.
var
dlp
=
DlpServiceClient
.
Create
();
//Construct the storage config by specifying the input directory.
var
storageConfig
=
new
StorageConfig
{
CloudStorageOptions
=
new
CloudStorageOptions
{
FileSet
=
new
CloudStorageOptions
.
Types
.
FileSet
{
Url
=
gcsInputPath
}
}
};
// Construct the inspect config by specifying the type of info to be inspected.
var
inspectConfig
=
new
InspectConfig
{
InfoTypes
=
{
new
InfoType
[]
{
new
InfoType
{
Name
=
"PERSON_NAME"
},
new
InfoType
{
Name
=
"EMAIL_ADDRESS"
}
}
},
IncludeQuote
=
true
};
// Construct the actions to take after the inspection portion of the job is completed.
// Specify how Cloud DLP must de-identify sensitive data in structured files, unstructured files and images
// using Transformation config.
// The de-identified files will be written to the the GCS bucket path specified in gcsOutputPath and the details of
// transformations performed will be written to BigQuery table specified in datasetId and tableId.
var
actions
=
new
Action
[]
{
new
Action
{
Deidentify
=
new
Action
.
Types
.
Deidentify
{
CloudStorageOutput
=
gcsOutputPath
,
TransformationConfig
=
new
TransformationConfig
{
DeidentifyTemplate
=
unstructuredDeidentifyTemplatePath
,
ImageRedactTemplate
=
imageRedactionTemplatePath
,
StructuredDeidentifyTemplate
=
structuredDeidentifyTemplatePath
,
},
TransformationDetailsStorageConfig
=
new
TransformationDetailsStorageConfig
{
Table
=
new
BigQueryTable
{
ProjectId
=
projectId
,
DatasetId
=
datasetId
,
TableId
=
tableId
}
}
}
}
};
// Construct the inspect job config using created storage config, inspect config and actions.
var
inspectJob
=
new
InspectJobConfig
{
StorageConfig
=
storageConfig
,
InspectConfig
=
inspectConfig
,
Actions
=
{
actions
}
};
// Create the dlp job and call the API.
DlpJob
response
=
dlp
.
CreateDlpJob
(
new
CreateDlpJobRequest
{
ParentAsLocationName
=
new
LocationName
(
projectId
,
"global"
),
InspectJob
=
inspectJob
});
return
response
;
}
}
Go
To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries .
To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment .
import
(
"context"
"fmt"
"io"
dlp
"cloud.google.com/go/dlp/apiv2"
"cloud.google.com/go/dlp/apiv2/dlppb"
)
func
deidentifyCloudStorage
(
w
io
.
Writer
,
projectID
,
gcsUri
,
tableId
,
datasetId
,
outputDirectory
,
deidentifyTemplateId
,
structuredDeidentifyTemplateId
,
imageRedactTemplateId
string
)
error
{
// projectId := "my-project-id"
// gcsUri := "gs://" + "your-bucket-name" + "/path/to/your/file.txt"
// tableId := "your-bigquery-table-id"
// datasetId := "your-bigquery-dataset-id"
// outputDirectory := "your-output-directory"
// deidentifyTemplateId := "your-deidentify-template-id"
// structuredDeidentifyTemplateId := "your-structured-deidentify-template-id"
// imageRedactTemplateId := "your-image-redact-template-id"
ctx
:=
context
.
Background
()
// Initialize a client once and reuse it to send multiple requests. Clients
// are safe to use across goroutines. When the client is no longer needed,
// call the Close method to cleanup its resources.
client
,
err
:=
dlp
.
NewClient
(
ctx
)
if
err
!=
nil
{
return
err
}
// Closing the client safely cleans up background resources.
defer
client
.
Close
()
// Set path in Cloud Storage.
cloudStorageOptions
:=
& dlppb
.
CloudStorageOptions
{
FileSet
:
& dlppb
.
CloudStorageOptions_FileSet
{
Url
:
gcsUri
,
},
}
// Define the storage config options for cloud storage options.
storageConfig
:=
& dlppb
.
StorageConfig
{
Type
:
& dlppb
.
StorageConfig_CloudStorageOptions
{
CloudStorageOptions
:
cloudStorageOptions
,
},
}
// Specify the type of info the inspection will look for.
// See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types
infoTypes
:=
[]
*
dlppb
.
InfoType
{
{
Name
:
"PERSON_NAME"
},
{
Name
:
"EMAIL_ADDRESS"
},
}
// inspectConfig holds the configuration settings for data inspection and analysis
// within the context of the Google Cloud Data Loss Prevention (DLP) API.
inspectConfig
:=
& dlppb
.
InspectConfig
{
InfoTypes
:
infoTypes
,
IncludeQuote
:
true
,
}
// Types of files to include for de-identification.
fileTypesToTransform
:=
[]
dlppb
.
FileType
{
dlppb
.
FileType_CSV
,
dlppb
.
FileType_IMAGE
,
dlppb
.
FileType_TEXT_FILE
,
}
// Specify the BigQuery table to be inspected.
table
:=
& dlppb
.
BigQueryTable
{
ProjectId
:
projectID
,
DatasetId
:
datasetId
,
TableId
:
tableId
,
}
// transformationDetailsStorageConfig holds configuration settings for storing transformation
// details in the context of the Google Cloud Data Loss Prevention (DLP) API.
transformationDetailsStorageConfig
:=
& dlppb
.
TransformationDetailsStorageConfig
{
Type
:
& dlppb
.
TransformationDetailsStorageConfig_Table
{
Table
:
table
,
},
}
transformationConfig
:=
& dlppb
.
TransformationConfig
{
DeidentifyTemplate
:
deidentifyTemplateId
,
ImageRedactTemplate
:
imageRedactTemplateId
,
StructuredDeidentifyTemplate
:
structuredDeidentifyTemplateId
,
}
// Action to execute on the completion of a job.
deidentify
:=
& dlppb
.
Action_Deidentify
{
TransformationConfig
:
transformationConfig
,
TransformationDetailsStorageConfig
:
transformationDetailsStorageConfig
,
Output
:
& dlppb
.
Action_Deidentify_CloudStorageOutput
{
CloudStorageOutput
:
outputDirectory
,
},
FileTypesToTransform
:
fileTypesToTransform
,
}
action
:=
& dlppb
.
Action
{
Action
:
& dlppb
.
Action_Deidentify_
{
Deidentify
:
deidentify
,
},
}
// Configure the inspection job we want the service to perform.
inspectJobConfig
:=
& dlppb
.
InspectJobConfig
{
StorageConfig
:
storageConfig
,
InspectConfig
:
inspectConfig
,
Actions
:
[]
*
dlppb
.
Action
{
action
,
},
}
// Construct the job creation request to be sent by the client.
req
:=
& dlppb
.
CreateDlpJobRequest
{
Parent
:
fmt
.
Sprintf
(
"projects/%s/locations/global"
,
projectID
),
Job
:
& dlppb
.
CreateDlpJobRequest_InspectJob
{
InspectJob
:
inspectJobConfig
,
},
}
// Send the request.
resp
,
err
:=
client
.
CreateDlpJob
(
ctx
,
req
)
if
err
!=
nil
{
fmt
.
Fprintf
(
w
,
"error after resp: %v"
,
err
)
return
err
}
// Print the results.
fmt
.
Fprint
(
w
,
"Job created successfully: "
,
resp
.
Name
)
return
nil
}
Java
To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries .
To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment .
import
com.google.cloud.dlp.v2. DlpServiceClient
;
import
com.google.privacy.dlp.v2. Action
;
import
com.google.privacy.dlp.v2. BigQueryTable
;
import
com.google.privacy.dlp.v2. CloudStorageOptions
;
import
com.google.privacy.dlp.v2. CreateDlpJobRequest
;
import
com.google.privacy.dlp.v2. DlpJob
;
import
com.google.privacy.dlp.v2. FileType
;
import
com.google.privacy.dlp.v2. InfoType
;
import
com.google.privacy.dlp.v2. InfoTypeStats
;
import
com.google.privacy.dlp.v2. InspectConfig
;
import
com.google.privacy.dlp.v2. InspectDataSourceDetails
;
import
com.google.privacy.dlp.v2. InspectJobConfig
;
import
com.google.privacy.dlp.v2. LocationName
;
import
com.google.privacy.dlp.v2. ProjectDeidentifyTemplateName
;
import
com.google.privacy.dlp.v2. StorageConfig
;
import
com.google.privacy.dlp.v2. TransformationConfig
;
import
com.google.privacy.dlp.v2. TransformationDetailsStorageConfig
;
import
java.io.IOException
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.List
;
import
java.util.concurrent.TimeUnit
;
public
class
DeidentifyCloudStorage
{
// Set the timeout duration in minutes.
private
static
final
int
TIMEOUT_MINUTES
=
15
;
public
static
void
main
(
String
[]
args
)
throws
IOException
,
InterruptedException
{
// TODO(developer): Replace these variables before running the sample.
// The Google Cloud project id to use as a parent resource.
String
projectId
=
"your-project-id"
;
// Specify the cloud storage directory that you want to inspect.
String
gcsPath
=
"gs://"
+
"your-bucket-name"
+
"/path/to/your/file.txt"
;
// Specify the big query dataset id to store the transformation details.
String
datasetId
=
"your-bigquery-dataset-id"
;
// Specify the big query table id to store the transformation details.
String
tableId
=
"your-bigquery-table-id"
;
// Specify the cloud storage directory to store the de-identified files.
String
outputDirectory
=
"your-output-directory"
;
// Specify the de-identify template ID for unstructured files.
String
deidentifyTemplateId
=
"your-deidentify-template-id"
;
// Specify the de-identify template ID for structured files.
String
structuredDeidentifyTemplateId
=
"your-structured-deidentify-template-id"
;
// Specify the de-identify template ID for images.
String
imageRedactTemplateId
=
"your-image-redact-template-id"
;
deidentifyCloudStorage
(
projectId
,
gcsPath
,
tableId
,
datasetId
,
outputDirectory
,
deidentifyTemplateId
,
structuredDeidentifyTemplateId
,
imageRedactTemplateId
);
}
public
static
void
deidentifyCloudStorage
(
String
projectId
,
String
gcsPath
,
String
tableId
,
String
datasetId
,
String
outputDirectory
,
String
deidentifyTemplateId
,
String
structuredDeidentifyTemplateId
,
String
imageRedactTemplateId
)
throws
IOException
,
InterruptedException
{
try
(
DlpServiceClient
dlp
=
DlpServiceClient
.
create
())
{
// Set path in Cloud Storage.
CloudStorageOptions
cloudStorageOptions
=
CloudStorageOptions
.
newBuilder
()
.
setFileSet
(
CloudStorageOptions
.
FileSet
.
newBuilder
().
setUrl
(
gcsPath
))
.
build
();
// Set storage config indicating the type of cloud storage.
StorageConfig
storageConfig
=
StorageConfig
.
newBuilder
().
setCloudStorageOptions
(
cloudStorageOptions
).
build
();
// Specify the type of info the inspection will look for.
// See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types
List<InfoType>
infoTypes
=
new
ArrayList
<> ();
for
(
String
typeName
:
new
String
[]
{
"PERSON_NAME"
,
"EMAIL_ADDRESS"
})
{
infoTypes
.
add
(
InfoType
.
newBuilder
().
setName
(
typeName
).
build
());
}
InspectConfig
inspectConfig
=
InspectConfig
.
newBuilder
().
addAllInfoTypes
(
infoTypes
).
setIncludeQuote
(
true
).
build
();
// Types of files to include for de-identification.
List<FileType>
fileTypesToTransform
=
Arrays
.
asList
(
FileType
.
valueOf
(
"IMAGE"
),
FileType
.
valueOf
(
"CSV"
),
FileType
.
valueOf
(
"TEXT_FILE"
));
// Specify the big query table to store the transformation details.
BigQueryTable
table
=
BigQueryTable
.
newBuilder
()
.
setProjectId
(
projectId
)
.
setTableId
(
tableId
)
.
setDatasetId
(
datasetId
)
.
build
();
TransformationDetailsStorageConfig
transformationDetailsStorageConfig
=
TransformationDetailsStorageConfig
.
newBuilder
().
setTable
(
table
).
build
();
// Specify the de-identify template used for the transformation.
TransformationConfig
transformationConfig
=
TransformationConfig
.
newBuilder
()
.
setDeidentifyTemplate
(
ProjectDeidentifyTemplateName
.
of
(
projectId
,
deidentifyTemplateId
).
toString
())
.
setImageRedactTemplate
(
ProjectDeidentifyTemplateName
.
of
(
projectId
,
imageRedactTemplateId
).
toString
())
.
setStructuredDeidentifyTemplate
(
ProjectDeidentifyTemplateName
.
of
(
projectId
,
structuredDeidentifyTemplateId
)
.
toString
())
.
build
();
Action
.
Deidentify
deidentify
=
Action
.
Deidentify
.
newBuilder
()
.
setCloudStorageOutput
(
outputDirectory
)
.
setTransformationConfig
(
transformationConfig
)
.
setTransformationDetailsStorageConfig
(
transformationDetailsStorageConfig
)
.
addAllFileTypesToTransform
(
fileTypesToTransform
)
.
build
();
Action
action
=
Action
.
newBuilder
().
setDeidentify
(
deidentify
).
build
();
// Configure the long-running job we want the service to perform.
InspectJobConfig
inspectJobConfig
=
InspectJobConfig
.
newBuilder
()
.
setInspectConfig
(
inspectConfig
)
.
setStorageConfig
(
storageConfig
)
.
addActions
(
action
)
.
build
();
// Construct the job creation request to be sent by the client.
CreateDlpJobRequest
createDlpJobRequest
=
CreateDlpJobRequest
.
newBuilder
()
.
setParent
(
LocationName
.
of
(
projectId
,
"global"
).
toString
())
.
setInspectJob
(
inspectJobConfig
)
.
build
();
// Send the job creation request.
DlpJob
response
=
dlp
.
createDlpJob
(
createDlpJobRequest
);
// Get the current time.
long
startTime
=
System
.
currentTimeMillis
();
// Check if the job state is DONE.
while
(
response
.
getState
()
!=
DlpJob
.
JobState
.
DONE
)
{
// Sleep for 30 second.
Thread
.
sleep
(
30000
);
// Get the updated job status.
response
=
dlp
.
getDlpJob
(
response
.
getName
());
// Check if the timeout duration has exceeded.
long
elapsedTime
=
System
.
currentTimeMillis
()
-
startTime
;
if
(
TimeUnit
.
MILLISECONDS
.
toMinutes
(
elapsedTime
)
> =
TIMEOUT_MINUTES
)
{
System
.
out
.
printf
(
"Job did not complete within %d minutes.%n"
,
TIMEOUT_MINUTES
);
break
;
}
}
// Print the results.
System
.
out
.
println
(
"Job status: "
+
response
.
getState
());
System
.
out
.
println
(
"Job name: "
+
response
.
getName
());
InspectDataSourceDetails
.
Result
result
=
response
.
getInspectDetails
().
getResult
();
System
.
out
.
println
(
"Findings: "
);
for
(
InfoTypeStats
infoTypeStat
:
result
.
getInfoTypeStatsList
())
{
System
.
out
.
print
(
"\tInfo type: "
+
infoTypeStat
.
getInfoType
().
getName
());
System
.
out
.
println
(
"\tCount: "
+
infoTypeStat
.
getCount
());
}
}
}
}
Node.js
To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries .
To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment .
// Imports the Google Cloud client library
const
DLP
=
require
(
' @google-cloud/dlp
'
);
// Instantiates a client
const
dlp
=
new
DLP
.
DlpServiceClient
();
// The project ID to run the API call under
// const projectId = 'my-project';
// The Cloud Storage directory that needs to be inspected
// const inputDirectory = 'your-google-cloud-storage-path';
// The ID of the dataset to inspect, e.g. 'my_dataset'
// const datasetId = 'my_dataset';
// The ID of the table to inspect, e.g. 'my_table'
// const tableId = 'my_table';
// The Cloud Storage directory that will be used to store the de-identified files
// const outputDirectory = 'your-output-directory';
// The full resource name of the default de-identify template
// const deidentifyTemplateId = 'your-deidentify-template-id';
// The full resource name of the de-identify template for structured files
// const structuredDeidentifyTemplateId = 'your-structured-deidentify-template-id';
// The full resource name of the image redaction template for images
// const imageRedactTemplateId = 'your-image-redact-template-id';
async
function
deidentifyCloudStorage
()
{
// Specify storage configuration that uses file set.
const
storageConfig
=
{
cloudStorageOptions
:
{
fileSet
:
{
url
:
inputDirectory
,
},
},
};
// Specify the type of info the inspection will look for.
const
infoTypes
=
[{
name
:
'PERSON_NAME'
},
{
name
:
'EMAIL_ADDRESS'
}];
// Construct inspect configuration
const
inspectConfig
=
{
infoTypes
:
infoTypes
,
includeQuote
:
true
,
};
// Types of files to include for de-identification.
const
fileTypesToTransform
=
[
{
fileType
:
'IMAGE'
},
{
fileType
:
'CSV'
},
{
fileType
:
' TEXT_FILE
'
},
];
// Specify the big query table to store the transformation details.
const
transformationDetailsStorageConfig
=
{
table
:
{
projectId
:
projectId
,
tableId
:
tableId
,
datasetId
:
datasetId
,
},
};
// Specify the de-identify template used for the transformation.
const
transformationConfig
=
{
deidentifyTemplate
:
deidentifyTemplateId
,
structuredDeidentifyTemplate
:
structuredDeidentifyTemplateId
,
imageRedactTemplate
:
imageRedactTemplateId
,
};
// Construct action to de-identify sensitive data.
const
action
=
{
deidentify
:
{
cloudStorageOutput
:
outputDirectory
,
transformationConfig
:
transformationConfig
,
transformationDetailsStorageConfig
:
transformationDetailsStorageConfig
,
fileTypes
:
fileTypesToTransform
,
},
};
// Construct the inspect job configuration.
const
inspectJobConfig
=
{
inspectConfig
:
inspectConfig
,
storageConfig
:
storageConfig
,
actions
:
[
action
],
};
// Construct the job creation request to be sent by the client.
const
createDlpJobRequest
=
{
parent
:
`projects/
${
projectId
}
/locations/global`
,
inspectJob
:
inspectJobConfig
,
};
// Send the job creation request and process the response.
const
[
response
]
=
await
dlp
.
createDlpJob
(
createDlpJobRequest
);
const
jobName
=
response
.
name
;
// Waiting for a maximum of 15 minutes for the job to get complete.
let
job
;
let
numOfAttempts
=
30
;
while
(
numOfAttempts
>
0
)
{
// Fetch DLP Job status
[
job
]
=
await
dlp
.
getDlpJob
({
name
:
jobName
});
// Check if the job has completed.
if
(
job
.
state
===
'DONE'
)
{
break
;
}
if
(
job
.
state
===
'FAILED'
)
{
console
.
log
(
'Job Failed, Please check the configuration.'
);
return
;
}
// Sleep for a short duration before checking the job status again.
await
new
Promise
(
resolve
=
>
{
setTimeout
(()
=
>
resolve
(),
30000
);
});
numOfAttempts
-=
1
;
}
// Print out the results.
const
infoTypeStats
=
job
.
inspectDetails
.
result
.
infoTypeStats
;
if
(
infoTypeStats
.
length
>
0
)
{
infoTypeStats
.
forEach
(
infoTypeStat
=
>
{
console
.
log
(
` Found
${
infoTypeStat
.
count
}
instance(s) of infoType
${
infoTypeStat
.
infoType
.
name
}
.`
);
});
}
else
{
console
.
log
(
'No findings.'
);
}
}
await
deidentifyCloudStorage
();
PHP
To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries .
To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment .
use Google\Cloud\Dlp\V2\Action;
use Google\Cloud\Dlp\V2\Action\Deidentify;
use Google\Cloud\Dlp\V2\BigQueryTable;
use Google\Cloud\Dlp\V2\Client\DlpServiceClient;
use Google\Cloud\Dlp\V2\CloudStorageOptions;
use Google\Cloud\Dlp\V2\CloudStorageOptions\FileSet;
use Google\Cloud\Dlp\V2\CreateDlpJobRequest;
use Google\Cloud\Dlp\V2\DlpJob\JobState;
use Google\Cloud\Dlp\V2\FileType;
use Google\Cloud\Dlp\V2\GetDlpJobRequest;
use Google\Cloud\Dlp\V2\InfoType;
use Google\Cloud\Dlp\V2\InspectConfig;
use Google\Cloud\Dlp\V2\InspectJobConfig;
use Google\Cloud\Dlp\V2\StorageConfig;
use Google\Cloud\Dlp\V2\TransformationConfig;
use Google\Cloud\Dlp\V2\TransformationDetailsStorageConfig;
/**
* De-identify sensitive data stored in Cloud Storage using the API.
* Create an inspection job that has a de-identification action.
*
* @param string $callingProjectId The project ID to run the API call under.
* @param string $inputgcsPath The Cloud Storage directory that you want to de-identify.
* @param string $outgcsPath The Cloud Storage directory where you want to store the
* de-identified files.
* @param string $deidentifyTemplateName The full resource name of the default de-identify template — for
* unstructured and structured files — if you created one. This value
* must be in the format
* `projects/projectName/(locations/locationId)/deidentifyTemplates/templateName`.
* @param string $structuredDeidentifyTemplateName The full resource name of the de-identify template for structured
* files if you created one. This value must be in the format
* `projects/projectName/(locations/locationId)/deidentifyTemplates/templateName`.
* @param string $imageRedactTemplateName The full resource name of the image redaction template for images if
* you created one. This value must be in the format
* `projects/projectName/(locations/locationId)/deidentifyTemplates/templateName`.
* @param string $datasetId The ID of the BigQuery dataset where you want to store
* the transformation details. If you don't provide a table ID, the
* system automatically creates one.
* @param string $tableId The ID of the BigQuery table where you want to store the
* transformation details.
*/
function deidentify_cloud_storage(
// TODO(developer): Replace sample parameters before running the code.
string $callingProjectId,
string $inputgcsPath = 'gs://YOUR_GOOGLE_STORAGE_BUCKET',
string $outgcsPath = 'gs://YOUR_GOOGLE_STORAGE_BUCKET',
string $deidentifyTemplateName = 'YOUR_DEIDENTIFY_TEMPLATE_NAME',
string $structuredDeidentifyTemplateName = 'YOUR_STRUCTURED_DEIDENTIFY_TEMPLATE_NAME',
string $imageRedactTemplateName = 'YOUR_IMAGE_REDACT_DEIDENTIFY_TEMPLATE_NAME',
string $datasetId = 'YOUR_DATASET_ID',
string $tableId = 'YOUR_TABLE_ID'
): void {
// Instantiate a client.
$dlp = new DlpServiceClient();
$parent = "projects/$callingProjectId/locations/global";
// Specify the GCS Path to be de-identify.
$cloudStorageOptions = (new CloudStorageOptions())
->setFileSet((new FileSet())
->setUrl($inputgcsPath));
$storageConfig = (new StorageConfig())
->setCloudStorageOptions(($cloudStorageOptions));
// Specify the type of info the inspection will look for.
$inspectConfig = (new InspectConfig())
->setInfoTypes([
(new InfoType())->setName('PERSON_NAME'),
(new InfoType())->setName('EMAIL_ADDRESS')
]);
// Specify the big query table to store the transformation details.
$transformationDetailsStorageConfig = (new TransformationDetailsStorageConfig())
->setTable((new BigQueryTable())
->setProjectId($callingProjectId)
->setDatasetId($datasetId)
->setTableId($tableId));
// Specify the de-identify template used for the transformation.
$transformationConfig = (new TransformationConfig())
->setDeidentifyTemplate(
DlpServiceClient::projectDeidentifyTemplateName($callingProjectId, $deidentifyTemplateName)
)
->setStructuredDeidentifyTemplate(
DlpServiceClient::projectDeidentifyTemplateName($callingProjectId, $structuredDeidentifyTemplateName)
)
->setImageRedactTemplate(
DlpServiceClient::projectDeidentifyTemplateName($callingProjectId, $imageRedactTemplateName)
);
$deidentify = (new Deidentify())
->setCloudStorageOutput($outgcsPath)
->setTransformationConfig($transformationConfig)
->setTransformationDetailsStorageConfig($transformationDetailsStorageConfig)
->setFileTypesToTransform([FileType::TEXT_FILE, FileType::IMAGE, FileType::CSV]);
$action = (new Action())
->setDeidentify($deidentify);
// Configure the inspection job we want the service to perform.
$inspectJobConfig = (new InspectJobConfig())
->setInspectConfig($inspectConfig)
->setStorageConfig($storageConfig)
->setActions([$action]);
// Send the job creation request and process the response.
$createDlpJobRequest = (new CreateDlpJobRequest())
->setParent($parent)
->setInspectJob($inspectJobConfig);
$job = $dlp->createDlpJob($createDlpJobRequest);
$numOfAttempts = 10;
do {
printf('Waiting for job to complete' . PHP_EOL);
sleep(30);
$getDlpJobRequest = (new GetDlpJobRequest())
->setName($job->getName());
$job = $dlp->getDlpJob($getDlpJobRequest);
if ($job->getState() == JobState::DONE) {
break;
}
$numOfAttempts--;
} while ($numOfAttempts > 0);
// Print finding counts.
printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
switch ($job->getState()) {
case JobState::DONE:
$infoTypeStats = $job->getInspectDetails()->getResult()->getInfoTypeStats();
if (count($infoTypeStats) === 0) {
printf('No findings.' . PHP_EOL);
} else {
foreach ($infoTypeStats as $infoTypeStat) {
printf(
' Found %s instance(s) of infoType %s' . PHP_EOL,
$infoTypeStat->getCount(),
$infoTypeStat->getInfoType()->getName()
);
}
}
break;
case JobState::FAILED:
printf('Job %s had errors:' . PHP_EOL, $job->getName());
$errors = $job->getErrors();
foreach ($errors as $error) {
var_dump($error->getDetails());
}
break;
case JobState::PENDING:
printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
break;
default:
printf('Unexpected job state. Most likely, the job is either running or has not yet started.');
}
}
Python
To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries .
To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment .
import
time
from
typing
import
List
import
google.cloud.dlp
def
deidentify_cloud_storage
(
project
:
str
,
input_gcs_bucket
:
str
,
output_gcs_bucket
:
str
,
info_types
:
List
[
str
],
deid_template_id
:
str
,
structured_deid_template_id
:
str
,
image_redact_template_id
:
str
,
dataset_id
:
str
,
table_id
:
str
,
timeout
:
int
=
300
,
)
-
> None
:
"""
Uses the Data Loss Prevention API to de-identify files in a Google Cloud
Storage directory.
Args:
project: The Google Cloud project id to use as a parent resource.
input_gcs_bucket: The name of google cloud storage bucket to inspect.
output_gcs_bucket: The name of google cloud storage bucket where
de-identified files would be stored.
info_types: A list of strings representing info types to look for.
A full list of info type categories can be fetched from the API.
deid_template_id: The name of the de-identify template for
unstructured and structured files.
structured_deid_template_id: The name of the de-identify template
for structured files.
image_redact_template_id: The name of the image redaction template
for images.
dataset_id: The identifier of the BigQuery dataset where transformation
details would be stored.
table_id: The identifier of the BigQuery table where transformation
details would be stored.
timeout: The number of seconds to wait for a response from the API.
"""
# Instantiate a client.
dlp
=
google
.
cloud
.
dlp_v2
.
DlpServiceClient
()
# Construct the configuration dictionary.
# Specify the type of info the inspection will look for.
# See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types.
inspect_config
=
{
"info_types"
:
[{
"name"
:
info_type
}
for
info_type
in
info_types
]}
# Construct cloud_storage_options dictionary with the bucket's URL.
storage_config
=
{
"cloud_storage_options"
:
{
"file_set"
:
{
"url"
:
f
"gs://
{
input_gcs_bucket
}
"
}}
}
# Specify the big query table to store the transformation details.
big_query_table
=
{
"project_id"
:
project
,
"dataset_id"
:
dataset_id
,
"table_id"
:
table_id
,
}
# Convert the project id into a full resource id.
parent
=
f
"projects/
{
project
}
/locations/global"
# Construct Transformation Configuration with de-identify Templates used
# for transformation.
transformation_config
=
{
"deidentify_template"
:
f
"
{
parent
}
/deidentifyTemplates/
{
deid_template_id
}
"
,
"structured_deidentify_template"
:
f
"
{
parent
}
/deidentifyTemplates/
{
structured_deid_template_id
}
"
,
"image_redact_template"
:
f
"
{
parent
}
/deidentifyTemplates/
{
image_redact_template_id
}
"
,
}
# Tell the API where to send notification when the job is completed.
actions
=
[
{
"deidentify"
:
{
"cloud_storage_output"
:
f
"gs://
{
output_gcs_bucket
}
"
,
"transformation_config"
:
transformation_config
,
"transformation_details_storage_config"
:
{
"table"
:
big_query_table
},
"file_types_to_transform"
:
[
"IMAGE"
,
"CSV"
,
"TEXT_FILE"
],
}
}
]
# Construct the job definition.
inspect_job
=
{
"inspect_config"
:
inspect_config
,
"storage_config"
:
storage_config
,
"actions"
:
actions
,
}
# Call the API.
response
=
dlp
.
create_dlp_job
(
request
=
{
"parent"
:
parent
,
"inspect_job"
:
inspect_job
,
}
)
job_name
=
response
.
name
print
(
f
"Inspection Job started :
{
job_name
}
"
)
# Waiting for the job to get completed.
job
=
dlp
.
get_dlp_job
(
request
=
{
"name"
:
job_name
})
# Since the sleep time is kept as 30s, number of calls would be timeout/30.
no_of_attempts
=
timeout
//
30
while
no_of_attempts
!=
0
:
# Check if the job has completed.
if
job
.
state
==
google
.
cloud
.
dlp_v2
.
DlpJob
.
JobState
.
DONE
:
break
if
job
.
state
==
google
.
cloud
.
dlp_v2
.
DlpJob
.
JobState
.
FAILED
:
print
(
"Job Failed, Please check the configuration."
)
break
# Sleep for a short duration before checking the job status again.
time
.
sleep
(
30
)
no_of_attempts
-=
1
# Get DLP job status.
job
=
dlp
.
get_dlp_job
(
request
=
{
"name"
:
job_name
})
if
job
.
state
!=
google
.
cloud
.
dlp_v2
.
DlpJob
.
JobState
.
DONE
:
print
(
f
"Job did not complete within
{
timeout
}
minutes."
)
return
# Print out the results.
print
(
f
"Job name:
{
job
.
name
}
"
)
result
=
job
.
inspect_details
.
result
print
(
f
"Processed Bytes:
{
result
.
processed_bytes
}
"
)
if
result
.
info_type_stats
:
for
stats
in
result
.
info_type_stats
:
print
(
f
"Info type:
{
stats
.
info_type
.
name
}
"
)
print
(
f
"Count:
{
stats
.
count
}
"
)
else
:
print
(
"No findings."
)
What's next
To search and filter code samples for other Google Cloud products, see the Google Cloud sample browser .
