// [END dlp_inspect_datastore] // [START dlp_inspect_gcs] public static object InspectGCS( string projectId, string minLikelihood, int maxFindings, bool includeQuote, IEnumerable <InfoType> infoTypes, string bucketName, string topicId, string subscriptionId) { var inspectJob = new InspectJobConfig { StorageConfig = new StorageConfig { CloudStorageOptions = new CloudStorageOptions { FileSet = new CloudStorageOptions.Types.FileSet { Url = $"gs://{bucketName}/*.txt" }, BytesLimitPerFile = 1073741824 }, }, InspectConfig = new InspectConfig { InfoTypes = { infoTypes }, ExcludeInfoTypes = false, IncludeQuote = includeQuote, Limits = new FindingLimits { MaxFindingsPerRequest = maxFindings }, MinLikelihood = (Likelihood)System.Enum.Parse(typeof(Likelihood), minLikelihood) }, Actions = { new Google.Cloud.Dlp.V2.Action { // Send results to Pub/Sub topic PubSub = new Google.Cloud.Dlp.V2.Action.Types.PublishToPubSub { Topic = topicId, } } } }; // Issue Create Dlp Job Request DlpServiceClient client = DlpServiceClient.Create(); var request = new CreateDlpJobRequest { InspectJob = inspectJob, ParentAsProjectName = new Google.Cloud.Dlp.V2.ProjectName(projectId), }; // We need created job name var dlpJob = client.CreateDlpJob(request); // Get a pub/sub subscription and listen for DLP results var fireEvent = new ManualResetEventSlim(); var subscriptionName = new SubscriptionName(projectId, subscriptionId); var subscriberClient = SubscriberServiceApiClient.Create(); var subscriber = SubscriberClient.Create(subscriptionName, new[] { subscriberClient }); subscriber.StartAsync( (pubSubMessage, cancellationToken) => { // Given a message that we receive on this subscription, we should either acknowledge or decline it if (pubSubMessage.Attributes["DlpJobName"] == dlpJob.Name) { fireEvent.Set(); return(Task.FromResult(SubscriberClient.Reply.Ack)); } return(Task.FromResult(SubscriberClient.Reply.Nack)); }); // We block here until receiving a signal from a separate thread that is waiting on a message indicating receiving a result of Dlp job if (fireEvent.Wait(TimeSpan.FromMinutes(1))) { // Stop the thread that is listening to messages as a result of StartAsync call earlier subscriber.StopAsync(CancellationToken.None).Wait(); // Now we can inspect full job results var job = client.GetDlpJob(new GetDlpJobRequest { DlpJobName = new DlpJobName(projectId, dlpJob.Name) }); // Inspect Job details Console.WriteLine($"Processed bytes: {job.InspectDetails.Result.ProcessedBytes}"); Console.WriteLine($"Total estimated bytes: {job.InspectDetails.Result.TotalEstimatedBytes}"); var stats = job.InspectDetails.Result.InfoTypeStats; Console.WriteLine("Found stats:"); foreach (var stat in stats) { Console.WriteLine($"{stat.InfoType.Name}"); } } else { Console.WriteLine("Error: The wait failed on timeout"); } return(0); }
public static object Inspect( string projectId, string minLikelihood, int maxFindings, bool includeQuote, string kindName, string namespaceId, IEnumerable <InfoType> infoTypes, IEnumerable <CustomInfoType> customInfoTypes, string datasetId, string tableId) { var inspectJob = new InspectJobConfig { StorageConfig = new StorageConfig { DatastoreOptions = new DatastoreOptions { Kind = new KindExpression { Name = kindName }, PartitionId = new PartitionId { NamespaceId = namespaceId, ProjectId = projectId, } }, TimespanConfig = new StorageConfig.Types.TimespanConfig { StartTime = Timestamp.FromDateTime(System.DateTime.UtcNow.AddYears(-1)), EndTime = Timestamp.FromDateTime(System.DateTime.UtcNow) } }, InspectConfig = new InspectConfig { InfoTypes = { infoTypes }, CustomInfoTypes = { customInfoTypes }, Limits = new FindingLimits { MaxFindingsPerRequest = maxFindings }, ExcludeInfoTypes = false, IncludeQuote = includeQuote, MinLikelihood = (Likelihood)System.Enum.Parse(typeof(Likelihood), minLikelihood) }, Actions = { new Google.Cloud.Dlp.V2.Action { // Save results in BigQuery Table SaveFindings = new Google.Cloud.Dlp.V2.Action.Types.SaveFindings { OutputConfig = new OutputStorageConfig { Table = new Google.Cloud.Dlp.V2.BigQueryTable { ProjectId = projectId, DatasetId = datasetId, TableId = tableId } } }, } } }; // Issue Create Dlp Job Request var client = DlpServiceClient.Create(); var request = new CreateDlpJobRequest { InspectJob = inspectJob, ParentAsProjectName = new ProjectName(projectId), }; // We need created job name var dlpJob = client.CreateDlpJob(request); var jobName = dlpJob.Name; // Make sure the job finishes before inspecting the results. // Alternatively, we can inspect results opportunistically, but // for testing purposes, we want consistent outcome var finishedJob = EnsureJobFinishes(projectId, jobName); var bigQueryClient = BigQueryClient.Create(projectId); var table = bigQueryClient.GetTable(datasetId, tableId); // Return only first page of 10 rows Console.WriteLine("DLP v2 Results:"); var firstPage = table.ListRows(new ListRowsOptions { StartIndex = 0, PageSize = 10 }); foreach (var item in firstPage) { Console.WriteLine($"\t {item[""]}"); } return(finishedJob); }
// [START dlp_create_trigger] public static object CreateJobTrigger( string projectId, string bucketName, string minLikelihood, int maxFindings, bool autoPopulateTimespan, int scanPeriod, IEnumerable <InfoType> infoTypes, string triggerId, string displayName, string description) { DlpServiceClient dlp = DlpServiceClient.Create(); var jobConfig = new InspectJobConfig { InspectConfig = new InspectConfig { MinLikelihood = (Likelihood)Enum.Parse( typeof(Likelihood), minLikelihood ), Limits = new FindingLimits { MaxFindingsPerRequest = maxFindings }, InfoTypes = { infoTypes } }, StorageConfig = new StorageConfig { CloudStorageOptions = new CloudStorageOptions { FileSet = new FileSet { Url = $"gs://{bucketName}/*" } }, TimespanConfig = new TimespanConfig { EnableAutoPopulationOfTimespanConfig = autoPopulateTimespan } } }; var jobTrigger = new JobTrigger { Triggers = { new Trigger { Schedule = new Schedule { RecurrencePeriodDuration = new Google.Protobuf.WellKnownTypes.Duration { Seconds = scanPeriod * 60 * 60 * 24 } } } }, InspectJob = jobConfig, Status = Status.Healthy, DisplayName = displayName, Description = description }; JobTrigger response = dlp.CreateJobTrigger( new CreateJobTriggerRequest { ParentAsProjectName = new ProjectName(projectId), JobTrigger = jobTrigger, TriggerId = triggerId }); Console.WriteLine($"Successfully created trigger {response.Name}"); return(0); }