Exemplo n.º 1
0
        // [END dlp_inspect_datastore]


        // [START dlp_inspect_gcs]

        public static object InspectGCS(
            string projectId,
            string minLikelihood,
            int maxFindings,
            bool includeQuote,
            IEnumerable <InfoType> infoTypes,
            string bucketName,
            string topicId,
            string subscriptionId)
        {
            var inspectJob = new InspectJobConfig
            {
                StorageConfig = new StorageConfig
                {
                    CloudStorageOptions = new CloudStorageOptions
                    {
                        FileSet = new CloudStorageOptions.Types.FileSet {
                            Url = $"gs://{bucketName}/*.txt"
                        },
                        BytesLimitPerFile = 1073741824
                    },
                },
                InspectConfig = new InspectConfig
                {
                    InfoTypes        = { infoTypes },
                    ExcludeInfoTypes = false,
                    IncludeQuote     = includeQuote,
                    Limits           = new FindingLimits
                    {
                        MaxFindingsPerRequest = maxFindings
                    },
                    MinLikelihood = (Likelihood)System.Enum.Parse(typeof(Likelihood), minLikelihood)
                },
                Actions =
                {
                    new Google.Cloud.Dlp.V2.Action
                    {
                        // Send results to Pub/Sub topic
                        PubSub = new Google.Cloud.Dlp.V2.Action.Types.PublishToPubSub
                        {
                            Topic = topicId,
                        }
                    }
                }
            };

            // Issue Create Dlp Job Request
            DlpServiceClient client = DlpServiceClient.Create();
            var request             = new CreateDlpJobRequest
            {
                InspectJob          = inspectJob,
                ParentAsProjectName = new Google.Cloud.Dlp.V2.ProjectName(projectId),
            };

            // We need created job name
            var dlpJob = client.CreateDlpJob(request);

            // Get a pub/sub subscription and listen for DLP results
            var fireEvent = new ManualResetEventSlim();

            var subscriptionName = new SubscriptionName(projectId, subscriptionId);
            var subscriberClient = SubscriberServiceApiClient.Create();
            var subscriber       = SubscriberClient.Create(subscriptionName, new[] { subscriberClient });

            subscriber.StartAsync(
                (pubSubMessage, cancellationToken) =>
            {
                // Given a message that we receive on this subscription, we should either acknowledge or decline it
                if (pubSubMessage.Attributes["DlpJobName"] == dlpJob.Name)
                {
                    fireEvent.Set();
                    return(Task.FromResult(SubscriberClient.Reply.Ack));
                }

                return(Task.FromResult(SubscriberClient.Reply.Nack));
            });

            // We block here until receiving a signal from a separate thread that is waiting on a message indicating receiving a result of Dlp job
            if (fireEvent.Wait(TimeSpan.FromMinutes(1)))
            {
                // Stop the thread that is listening to messages as a result of StartAsync call earlier
                subscriber.StopAsync(CancellationToken.None).Wait();

                // Now we can inspect full job results
                var job = client.GetDlpJob(new GetDlpJobRequest {
                    DlpJobName = new DlpJobName(projectId, dlpJob.Name)
                });

                // Inspect Job details
                Console.WriteLine($"Processed bytes: {job.InspectDetails.Result.ProcessedBytes}");
                Console.WriteLine($"Total estimated bytes: {job.InspectDetails.Result.TotalEstimatedBytes}");
                var stats = job.InspectDetails.Result.InfoTypeStats;
                Console.WriteLine("Found stats:");
                foreach (var stat in stats)
                {
                    Console.WriteLine($"{stat.InfoType.Name}");
                }
            }
            else
            {
                Console.WriteLine("Error: The wait failed on timeout");
            }

            return(0);
        }
Exemplo n.º 2
0
    public static object Inspect(
        string projectId,
        string minLikelihood,
        int maxFindings,
        bool includeQuote,
        string kindName,
        string namespaceId,
        IEnumerable <InfoType> infoTypes,
        IEnumerable <CustomInfoType> customInfoTypes,
        string datasetId,
        string tableId)
    {
        var inspectJob = new InspectJobConfig
        {
            StorageConfig = new StorageConfig
            {
                DatastoreOptions = new DatastoreOptions
                {
                    Kind = new KindExpression {
                        Name = kindName
                    },
                    PartitionId = new PartitionId
                    {
                        NamespaceId = namespaceId,
                        ProjectId   = projectId,
                    }
                },
                TimespanConfig = new StorageConfig.Types.TimespanConfig
                {
                    StartTime = Timestamp.FromDateTime(System.DateTime.UtcNow.AddYears(-1)),
                    EndTime   = Timestamp.FromDateTime(System.DateTime.UtcNow)
                }
            },

            InspectConfig = new InspectConfig
            {
                InfoTypes       = { infoTypes },
                CustomInfoTypes = { customInfoTypes },
                Limits          = new FindingLimits
                {
                    MaxFindingsPerRequest = maxFindings
                },
                ExcludeInfoTypes = false,
                IncludeQuote     = includeQuote,
                MinLikelihood    = (Likelihood)System.Enum.Parse(typeof(Likelihood), minLikelihood)
            },
            Actions =
            {
                new Google.Cloud.Dlp.V2.Action
                {
                    // Save results in BigQuery Table
                    SaveFindings = new Google.Cloud.Dlp.V2.Action.Types.SaveFindings
                    {
                        OutputConfig = new OutputStorageConfig
                        {
                            Table = new Google.Cloud.Dlp.V2.BigQueryTable
                            {
                                ProjectId = projectId,
                                DatasetId = datasetId,
                                TableId   = tableId
                            }
                        }
                    },
                }
            }
        };

        // Issue Create Dlp Job Request
        var client  = DlpServiceClient.Create();
        var request = new CreateDlpJobRequest
        {
            InspectJob          = inspectJob,
            ParentAsProjectName = new ProjectName(projectId),
        };

        // We need created job name
        var dlpJob  = client.CreateDlpJob(request);
        var jobName = dlpJob.Name;

        // Make sure the job finishes before inspecting the results.
        // Alternatively, we can inspect results opportunistically, but
        // for testing purposes, we want consistent outcome
        var finishedJob    = EnsureJobFinishes(projectId, jobName);
        var bigQueryClient = BigQueryClient.Create(projectId);
        var table          = bigQueryClient.GetTable(datasetId, tableId);

        // Return only first page of 10 rows
        Console.WriteLine("DLP v2 Results:");
        var firstPage = table.ListRows(new ListRowsOptions {
            StartIndex = 0, PageSize = 10
        });

        foreach (var item in firstPage)
        {
            Console.WriteLine($"\t {item[""]}");
        }

        return(finishedJob);
    }
Exemplo n.º 3
0
        // [START dlp_create_trigger]
        public static object CreateJobTrigger(
            string projectId,
            string bucketName,
            string minLikelihood,
            int maxFindings,
            bool autoPopulateTimespan,
            int scanPeriod,
            IEnumerable <InfoType> infoTypes,
            string triggerId,
            string displayName,
            string description)
        {
            DlpServiceClient dlp = DlpServiceClient.Create();

            var jobConfig = new InspectJobConfig
            {
                InspectConfig = new InspectConfig
                {
                    MinLikelihood = (Likelihood)Enum.Parse(
                        typeof(Likelihood),
                        minLikelihood
                        ),
                    Limits = new FindingLimits
                    {
                        MaxFindingsPerRequest = maxFindings
                    },
                    InfoTypes = { infoTypes }
                },
                StorageConfig = new StorageConfig
                {
                    CloudStorageOptions = new CloudStorageOptions
                    {
                        FileSet = new FileSet
                        {
                            Url = $"gs://{bucketName}/*"
                        }
                    },
                    TimespanConfig = new TimespanConfig
                    {
                        EnableAutoPopulationOfTimespanConfig = autoPopulateTimespan
                    }
                }
            };

            var jobTrigger = new JobTrigger
            {
                Triggers =
                {
                    new Trigger
                    {
                        Schedule = new Schedule
                        {
                            RecurrencePeriodDuration = new Google.Protobuf.WellKnownTypes.Duration
                            {
                                Seconds = scanPeriod * 60 * 60 * 24
                            }
                        }
                    }
                },
                InspectJob  = jobConfig,
                Status      = Status.Healthy,
                DisplayName = displayName,
                Description = description
            };

            JobTrigger response = dlp.CreateJobTrigger(
                new CreateJobTriggerRequest
            {
                ParentAsProjectName = new ProjectName(projectId),
                JobTrigger          = jobTrigger,
                TriggerId           = triggerId
            });

            Console.WriteLine($"Successfully created trigger {response.Name}");
            return(0);
        }