// [END dlp_inspect_datastore]
// [START dlp_inspect_gcs]
public static object InspectGCS(
string projectId,
string minLikelihood,
int maxFindings,
bool includeQuote,
IEnumerable <InfoType> infoTypes,
string bucketName,
string topicId,
string subscriptionId)
{
var inspectJob = new InspectJobConfig
{
StorageConfig = new StorageConfig
{
CloudStorageOptions = new CloudStorageOptions
{
FileSet = new CloudStorageOptions.Types.FileSet {
Url = $"gs://{bucketName}/*.txt"
},
BytesLimitPerFile = 1073741824
},
},
InspectConfig = new InspectConfig
{
InfoTypes = { infoTypes },
ExcludeInfoTypes = false,
IncludeQuote = includeQuote,
Limits = new FindingLimits
{
MaxFindingsPerRequest = maxFindings
},
MinLikelihood = (Likelihood)System.Enum.Parse(typeof(Likelihood), minLikelihood)
},
Actions =
{
new Google.Cloud.Dlp.V2.Action
{
// Send results to Pub/Sub topic
PubSub = new Google.Cloud.Dlp.V2.Action.Types.PublishToPubSub
{
Topic = topicId,
}
}
}
};
// Issue Create Dlp Job Request
DlpServiceClient client = DlpServiceClient.Create();
var request = new CreateDlpJobRequest
{
InspectJob = inspectJob,
ParentAsProjectName = new Google.Cloud.Dlp.V2.ProjectName(projectId),
};
// We need created job name
var dlpJob = client.CreateDlpJob(request);
// Get a pub/sub subscription and listen for DLP results
var fireEvent = new ManualResetEventSlim();
var subscriptionName = new SubscriptionName(projectId, subscriptionId);
var subscriberClient = SubscriberServiceApiClient.Create();
var subscriber = SubscriberClient.Create(subscriptionName, new[] { subscriberClient });
subscriber.StartAsync(
(pubSubMessage, cancellationToken) =>
{
// Given a message that we receive on this subscription, we should either acknowledge or decline it
if (pubSubMessage.Attributes["DlpJobName"] == dlpJob.Name)
{
fireEvent.Set();
return(Task.FromResult(SubscriberClient.Reply.Ack));
}
return(Task.FromResult(SubscriberClient.Reply.Nack));
});
// We block here until receiving a signal from a separate thread that is waiting on a message indicating receiving a result of Dlp job
if (fireEvent.Wait(TimeSpan.FromMinutes(1)))
{
// Stop the thread that is listening to messages as a result of StartAsync call earlier
subscriber.StopAsync(CancellationToken.None).Wait();
// Now we can inspect full job results
var job = client.GetDlpJob(new GetDlpJobRequest {
DlpJobName = new DlpJobName(projectId, dlpJob.Name)
});
// Inspect Job details
Console.WriteLine($"Processed bytes: {job.InspectDetails.Result.ProcessedBytes}");
Console.WriteLine($"Total estimated bytes: {job.InspectDetails.Result.TotalEstimatedBytes}");
var stats = job.InspectDetails.Result.InfoTypeStats;
Console.WriteLine("Found stats:");
foreach (var stat in stats)
{
Console.WriteLine($"{stat.InfoType.Name}");
}
}
else
{
Console.WriteLine("Error: The wait failed on timeout");
}
return(0);
}
public static object Inspect(
string projectId,
string minLikelihood,
int maxFindings,
bool includeQuote,
string kindName,
string namespaceId,
IEnumerable <InfoType> infoTypes,
IEnumerable <CustomInfoType> customInfoTypes,
string datasetId,
string tableId)
{
var inspectJob = new InspectJobConfig
{
StorageConfig = new StorageConfig
{
DatastoreOptions = new DatastoreOptions
{
Kind = new KindExpression {
Name = kindName
},
PartitionId = new PartitionId
{
NamespaceId = namespaceId,
ProjectId = projectId,
}
},
TimespanConfig = new StorageConfig.Types.TimespanConfig
{
StartTime = Timestamp.FromDateTime(System.DateTime.UtcNow.AddYears(-1)),
EndTime = Timestamp.FromDateTime(System.DateTime.UtcNow)
}
},
InspectConfig = new InspectConfig
{
InfoTypes = { infoTypes },
CustomInfoTypes = { customInfoTypes },
Limits = new FindingLimits
{
MaxFindingsPerRequest = maxFindings
},
ExcludeInfoTypes = false,
IncludeQuote = includeQuote,
MinLikelihood = (Likelihood)System.Enum.Parse(typeof(Likelihood), minLikelihood)
},
Actions =
{
new Google.Cloud.Dlp.V2.Action
{
// Save results in BigQuery Table
SaveFindings = new Google.Cloud.Dlp.V2.Action.Types.SaveFindings
{
OutputConfig = new OutputStorageConfig
{
Table = new Google.Cloud.Dlp.V2.BigQueryTable
{
ProjectId = projectId,
DatasetId = datasetId,
TableId = tableId
}
}
},
}
}
};
// Issue Create Dlp Job Request
var client = DlpServiceClient.Create();
var request = new CreateDlpJobRequest
{
InspectJob = inspectJob,
ParentAsProjectName = new ProjectName(projectId),
};
// We need created job name
var dlpJob = client.CreateDlpJob(request);
var jobName = dlpJob.Name;
// Make sure the job finishes before inspecting the results.
// Alternatively, we can inspect results opportunistically, but
// for testing purposes, we want consistent outcome
var finishedJob = EnsureJobFinishes(projectId, jobName);
var bigQueryClient = BigQueryClient.Create(projectId);
var table = bigQueryClient.GetTable(datasetId, tableId);
// Return only first page of 10 rows
Console.WriteLine("DLP v2 Results:");
var firstPage = table.ListRows(new ListRowsOptions {
StartIndex = 0, PageSize = 10
});
foreach (var item in firstPage)
{
Console.WriteLine($"\t {item[""]}");
}
return(finishedJob);
}
// [START dlp_create_trigger]
public static object CreateJobTrigger(
string projectId,
string bucketName,
string minLikelihood,
int maxFindings,
bool autoPopulateTimespan,
int scanPeriod,
IEnumerable <InfoType> infoTypes,
string triggerId,
string displayName,
string description)
{
DlpServiceClient dlp = DlpServiceClient.Create();
var jobConfig = new InspectJobConfig
{
InspectConfig = new InspectConfig
{
MinLikelihood = (Likelihood)Enum.Parse(
typeof(Likelihood),
minLikelihood
),
Limits = new FindingLimits
{
MaxFindingsPerRequest = maxFindings
},
InfoTypes = { infoTypes }
},
StorageConfig = new StorageConfig
{
CloudStorageOptions = new CloudStorageOptions
{
FileSet = new FileSet
{
Url = $"gs://{bucketName}/*"
}
},
TimespanConfig = new TimespanConfig
{
EnableAutoPopulationOfTimespanConfig = autoPopulateTimespan
}
}
};
var jobTrigger = new JobTrigger
{
Triggers =
{
new Trigger
{
Schedule = new Schedule
{
RecurrencePeriodDuration = new Google.Protobuf.WellKnownTypes.Duration
{
Seconds = scanPeriod * 60 * 60 * 24
}
}
}
},
InspectJob = jobConfig,
Status = Status.Healthy,
DisplayName = displayName,
Description = description
};
JobTrigger response = dlp.CreateJobTrigger(
new CreateJobTriggerRequest
{
ParentAsProjectName = new ProjectName(projectId),
JobTrigger = jobTrigger,
TriggerId = triggerId
});
Console.WriteLine($"Successfully created trigger {response.Name}");
return(0);
}