private void InitializeQueue()
{
if (_sub == null)
{
var googleCredential = GoogleCredential.FromFile(_gcpSettings.PubSub.JsonAuthPath)
.CreateScoped(SubscriberServiceApiClient.DefaultScopes);
var channel = new Channel(SubscriberServiceApiClient.DefaultEndpoint.ToString(), googleCredential.ToChannelCredentials());
_sub = SubscriberServiceApiClient.Create(channel);
}
var queueName = _gcpSettings.PubSub.QueueMappings[typeof(T).FullName];
if (_topicName == null)
{
_topicName = new TopicName(_gcpSettings.PubSub.ProjectId, queueName);
}
if (_subscriptionName == null)
{
_subscriptionName = new SubscriptionName(_gcpSettings.PubSub.ProjectId, queueName);
}
}
public static object CreateSubscription(string projectId, string topicId,
string subscriptionId)
{
// [START pubsub_create_pull_subscription]
SubscriberServiceApiClient subscriber = SubscriberServiceApiClient.Create();
TopicName topicName = new TopicName(projectId, topicId);
SubscriptionName subscriptionName = new SubscriptionName(projectId,
subscriptionId);
try
{
Subscription subscription = subscriber.CreateSubscription(
subscriptionName, topicName, pushConfig: null,
ackDeadlineSeconds: 60);
}
catch (RpcException e)
when(e.Status.StatusCode == StatusCode.AlreadyExists)
{
// Already exists. That's fine.
}
// [END pubsub_create_pull_subscription]
return(0);
}
public async Task <int> PullMessagesAsync(string projectId, string subscriptionId, bool acknowledge)
{
SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(projectId, subscriptionId);
SubscriberClient subscriber = await SubscriberClient.CreateAsync(subscriptionName);
// SubscriberClient runs your message handle function on multiple
// threads to maximize throughput.
int messageCount = 0;
Task startTask = subscriber.StartAsync(
async(PubsubMessage message, CancellationToken cancel) =>
{
string text = Encoding.UTF8.GetString(message.Data.ToArray());
Console.WriteLine($"Message {message.MessageId}: {text}");
Interlocked.Increment(ref messageCount);
return(acknowledge ? SubscriberClient.Reply.Ack : SubscriberClient.Reply.Nack);
});
// Run for 5 seconds.
await Task.Delay(5000);
await subscriber.StopAsync(CancellationToken.None);
return(messageCount);
}
private async Task <SubscriptionSettings> GetSettings(HttpRequest req)
{
using StreamReader streamReader = new StreamReader(req.Body);
string body = await streamReader.ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(body);
SubscriptionName subscriptionName = ParseEnum <SubscriptionName>(data.SubscriptionName.ToString());
string url = data.WebhookUrl?.ToString();
if (!Uri.TryCreate(url, UriKind.Absolute, out _))
{
throw new ArgumentException($"Webhook URL seems invalid: {url}");
}
IntegrationPlatform platform = this.GetPlatform(url);
return(new SubscriptionSettings()
{
SubscriptionId = Guid.NewGuid().ToString(),
SubscriptionName = subscriptionName,
IntegrationPlatform = platform,
WebhookUrl = url
});
}
/// <summary>
/// Method to run Health Check for PubSub light Work
/// </summary>
/// <param name="serviceKey"></param>
/// <param name="value"></param>
/// <returns></returns>
public LightHealth.HealthCheck HealthCheck(string serviceKey, string value)
{
try
{
if (_topics.Count > 0)
{
var topicEnum = _topics.GetEnumerator();
var topic = topicEnum.Current;
GooglePubSubConfiguration config = GetConnection(topic);
MethodInfo method = GetMethod(topic);
string topicName = topic.Value.TopicName;
string subscriptName = topic.Value.Subscription;
this._subscriptionName = new SubscriptionName(config.ProjectID, subscriptName);
this.subscriberClient = SubscriberClient.CreateAsync(_subscriptionName).Result;
}
return(LightHealth.HealthCheck.Healthy);
}
catch
{
return(LightHealth.HealthCheck.Unhealthy);
}
}
/// <summary>
/// Create a subscriber with custom control flow settings.
/// </summary>
static SimpleSubscriber GetCustomSubscriber(string projectId,
string subscriptionId)
{
// [START pubsub_subscriber_flow_settings]
SubscriptionName subscriptionName = new SubscriptionName(projectId,
subscriptionId);
SubscriberClient subscriberClient = SubscriberClient.Create();
SimpleSubscriber subscriber = SimpleSubscriber.Create(
subscriptionName, new[] { subscriberClient },
new SimpleSubscriber.Settings()
{
AckExtensionWindow = TimeSpan.FromSeconds(4),
Scheduler = Google.Api.Gax.SystemScheduler.Instance,
StreamAckDeadline = TimeSpan.FromSeconds(10),
FlowControlSettings = new Google.Api.Gax
.FlowControlSettings(
maxOutstandingElementCount: 100,
maxOutstandardByteCount: 10240)
});
// [END pubsub_subscriber_flow_settings]
return(subscriber);
}
// [START retry]
internal void RpcRetry(string topicId, string subscriptionId,
PublisherServiceApiClient publisher, SubscriberServiceApiClient subscriber)
{
TopicName topicName = new TopicName(_projectId, topicId);
// Create Subscription.
SubscriptionName subscriptionName = new SubscriptionName(_projectId,
subscriptionId);
// Create Topic
try
{
// This may fail if the Topic already exists.
// Don't retry in that case.
publisher.CreateTopic(topicName, newRetryCallSettings(3,
StatusCode.AlreadyExists));
}
catch (RpcException e)
when(e.Status.StatusCode == StatusCode.AlreadyExists)
{
// Already exists. That's fine.
}
try
{
// Subscribe to Topic
// This may fail if the Subscription already exists. Don't
// retry, because a retry would fail the same way.
subscriber.CreateSubscription(subscriptionName, topicName,
pushConfig: null, ackDeadlineSeconds: 60,
callSettings: newRetryCallSettings(3,
StatusCode.AlreadyExists));
}
catch (RpcException e)
when(e.Status.StatusCode == StatusCode.AlreadyExists)
{
// Already exists. That's fine.
}
}
/// <summary>
/// Creates a Google PubSub subscription service
/// </summary>
/// <param name="projectId">GCP project ID</param>
/// <param name="subscriptionId">Google PubSub subscription ID (within the project), which must already exist</param>
/// <param name="eventSerializer">Event serializer instance</param>
/// <param name="eventHandlers">Collection of event handlers</param>
/// <param name="loggerFactory">Optional: logger factory</param>
/// <param name="measure">Callback for measuring the subscription gap</param>
public GooglePubSubSubscription(
string projectId,
string subscriptionId,
IEventSerializer eventSerializer,
IEnumerable <IEventHandler> eventHandlers,
ILoggerFactory?loggerFactory = null,
SubscriptionGapMeasure?measure = null
) : base(
subscriptionId,
new NoOpCheckpointStore(),
eventSerializer,
eventHandlers,
loggerFactory,
measure
)
{
_subscriptionName = SubscriptionName.FromProjectSubscription(
Ensure.NotEmptyString(projectId, nameof(projectId)),
Ensure.NotEmptyString(subscriptionId, nameof(subscriptionId))
);
_client = SubscriberClient.Create(_subscriptionName);
_metricClient = MetricServiceClient.Create();
_undeliveredCountRequest = GetFilteredRequest(PubSubMetricUndeliveredMessagesCount);
_oldestAgeRequest = GetFilteredRequest(PubSubMetricOldestUnackedMessageAge);
ListTimeSeriesRequest GetFilteredRequest(string metric)
=> new()
{
Name = $"projects/{projectId}",
Filter = $"metric.type = \"pubsub.googleapis.com/subscription/{metric}\" "
+ $"AND resource.label.subscription_id = \"{subscriptionId}\""
};
}
Task _subscriberTask = null !;
public int PullMessageWithLeaseManagement(string projectId, string subscriptionId, bool acknowledge)
{
SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(projectId, subscriptionId);
SubscriberServiceApiClient subscriberClient = SubscriberServiceApiClient.Create();
var ackIds = new List <string>();
try
{
PullResponse response = subscriberClient.Pull(subscriptionName, returnImmediately: false, maxMessages: 20);
// Print out each received message.
foreach (ReceivedMessage msg in response.ReceivedMessages)
{
ackIds.Add(msg.AckId);
string text = System.Text.Encoding.UTF8.GetString(msg.Message.Data.ToArray());
Console.WriteLine($"Message {msg.Message.MessageId}: {text}");
// Modify the ack deadline of each received message from the default 10 seconds to 30.
// This prevents the server from redelivering the message after the default 10 seconds
// have passed.
subscriberClient.ModifyAckDeadline(subscriptionName, new List <string> {
msg.AckId
}, 30);
}
// If acknowledgement required, send to server.
if (acknowledge && ackIds.Count > 0)
{
subscriberClient.Acknowledge(subscriptionName, ackIds);
}
}
catch (RpcException ex) when(ex.Status.StatusCode == StatusCode.Unavailable)
{
// UNAVAILABLE due to too many concurrent pull requests pending for the given subscription.
}
return(ackIds.Count);
}
public IEnumerable <ListaRetorno> ShowMessagesForSubscription([FromBody] CreatTopicModel attr)
{
List <ListaRetorno> retorno = new List <ListaRetorno>();
var subscriptionName = new SubscriptionName(attr.projectId, attr.subscriptionID);
var subscription = SubscriberServiceApiClient.Create();
try
{
PullResponse response = subscription.Pull(subscriptionName, true, 10);
var all = response.ReceivedMessages;
foreach (ReceivedMessage message in all)
{
retorno.Add(new ListaRetorno
{
id = message.Message.MessageId,
publishDate = message.Message.PublishTime.ToDateTime().ToString("dd-M-yyyy HH:MM:ss"),
data = Encoding.UTF8.GetString(message.Message.Data.ToArray(), 0, message.Message.Data.Length),
});
subscription.Acknowledge(subscriptionName, new string[] { message.AckId });
}
return(retorno);
}
catch (RpcException ex)
{
Console.WriteLine("Erro: {0}", ex.Message);
retorno.Add(new ListaRetorno {
data = ex.Message
});
return(retorno);
}
}
public PubSubService(
ILogger <PubSubService> logger,
IConfiguration configuration
)
{
_logger = logger;
/// This may depend on how your application is hosted.
/// But if the application is served through kestrel, the credentials are usually found in the
/// same folder as the published dlls.
_pubSubCredentials = configuration["GCP:Credentials"];
_pubSubCredentials = Directory.GetCurrentDirectory() + $"/{_pubSubCredentials}";
/// Setup GCP variables
var projectId = configuration["GCP:ProjectID"];
var userCreateTopicId = configuration["GCP:Topics:createUser"];
var userCreateSubscriptionId = configuration["GCP:Subscriptions:createUser"];
/// Create the topic names
_userCreationTopic = new TopicName(projectId, userCreateTopicId);
/// Create the subscription names
_userCreationSubscription = new SubscriptionName(projectId, userCreateSubscriptionId);
}
/// <summary>
/// Initializes a new instance of the <see cref="Subscription"/> class.
/// </summary>
/// <param name="dataType">Type of the data.</param>
/// <param name="name">The name. Defaults to the data type's full name.</param>
/// <param name="channelName">The channel name. Defaults to the data type's full name.</param>
/// <param name="routingKey">The routing key. Defaults to the data type's full name.</param>
/// <param name="groupId">What is the id of the consumer group that this consumer belongs to; will not process the same partition as others in group</param>
/// <param name="bufferSize">The number of messages to buffer at any one time, also the number of messages to retrieve at once. Min of 1 Max of 10</param>
/// <param name="noOfPerformers">The no of threads reading this channel.</param>
/// <param name="timeoutInMilliseconds">The timeout in milliseconds.</param>
/// <param name="requeueCount">The number of times you want to requeue a message before dropping it.</param>
/// <param name="requeueDelayInMilliseconds">The number of milliseconds to delay the delivery of a requeue message for.</param>
/// <param name="unacceptableMessageLimit">The number of unacceptable messages to handle, before stopping reading from the channel.</param>
/// <param name="offsetDefault">Where should we begin processing if we cannot find a stored offset</param>
/// <param name="commitBatchSize">How often should we commit offsets?</param>
/// <param name="sessionTimeoutMs">What is the heartbeat interval for this consumer, after which Kafka will assume dead and rebalance the consumer group</param>
/// <param name="maxPollIntervalMs">How often does the consumer poll for a message to be considered alive, after which Kafka will assume dead and rebalance</param>
/// <param name="sweepUncommittedOffsetsIntervalMs">How often do we commit offsets that have yet to be saved</param>
/// <param name="isolationLevel">Should we read messages that are not on all replicas? May cause duplicates.</param>
/// <param name="isAsync">Is this channel read asynchronously</param>
/// <param name="numOfPartitions">How many partitions should this topic have - used if we create the topic</param>
/// <param name="replicationFactor">How many copies of each partition should we have across our broker's nodes - used if we create the topic</param> /// <param name="channelFactory">The channel factory to create channels for Consumer.</param>
/// <param name="makeChannels">Should we make channels if they don't exist, defaults to creating</param>
public KafkaSubscription(
Type dataType,
SubscriptionName name = null,
ChannelName channelName = null,
RoutingKey routingKey = null,
string groupId = null,
int bufferSize = 1,
int noOfPerformers = 1,
int timeoutInMilliseconds = 300,
int requeueCount = -1,
int requeueDelayInMilliseconds = 0,
int unacceptableMessageLimit = 0,
AutoOffsetReset offsetDefault = AutoOffsetReset.Earliest,
long commitBatchSize = 10,
int sessionTimeoutMs = 10000,
int maxPollIntervalMs = 300000,
int sweepUncommittedOffsetsIntervalMs = 30000,
IsolationLevel isolationLevel = IsolationLevel.ReadCommitted,
bool isAsync = false,
int numOfPartitions = 1,
short replicationFactor = 1,
IAmAChannelFactory channelFactory = null,
OnMissingChannel makeChannels = OnMissingChannel.Create)
: base(dataType, name, channelName, routingKey, bufferSize, noOfPerformers, timeoutInMilliseconds, requeueCount,
requeueDelayInMilliseconds, unacceptableMessageLimit, isAsync, channelFactory, makeChannels)
{
CommitBatchSize = commitBatchSize;
GroupId = groupId;
IsolationLevel = isolationLevel;
MaxPollIntervalMs = maxPollIntervalMs;
SweepUncommittedOffsetsIntervalMs = sweepUncommittedOffsetsIntervalMs;
OffsetDefault = offsetDefault;
SessionTimeoutMs = sessionTimeoutMs;
NumPartitions = numOfPartitions;
ReplicationFactor = replicationFactor;
}
public static IComicRetriever Get(SubscriptionName subscriptionName)
{
switch (subscriptionName)
{
case SubscriptionName.CommitStripOfTheDay:
return(new Retrievers.CommitStrip.ComicOfTheDayRetriever());
case SubscriptionName.CommitStripRandom:
return(new Retrievers.CommitStrip.RandomRetriever());
case SubscriptionName.MonkeyUserRandom:
return(new Retrievers.MonkeyUser.RandomRetriever());
//dilbert - not officially supported as no confirmation from the authors
case SubscriptionName.DilbertRandom:
return(new Retrievers.Dilbert.RandomRetriever());
case SubscriptionName.DilbertOfTheDay:
return(new Retrievers.Dilbert.ComicOfTheDayRetriever());
default:
throw new ArgumentOutOfRangeException(nameof(subscriptionName), subscriptionName, $"Retriever for {subscriptionName} not implemented.");
}
}
private async Task StartConsumingAsync(string projectId, string subscriptionId, IMessageHandler messageHandler, bool throttle)
{
var subscriptionName = new SubscriptionName(projectId, subscriptionId);
var settings = throttle
? new SubscriberClient.Settings {
FlowControlSettings = new FlowControlSettings(1L, null)
}
: null;
_subscriberClient = await SubscriberClient.CreateAsync(subscriptionName, null, settings);
//Blame google
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
_subscriberClient.StartAsync((msg, cancellationToken) =>
{
var message = msg.Data.ToStringUtf8();
return(Task.FromResult(messageHandler.HandleMessage(message)
? SubscriberClient.Reply.Ack
: SubscriberClient.Reply.Nack));
}).ConfigureAwait(false);
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
}
/// <summary>
/// The operation to rename a subscription
/// </summary>
/// <param name='subscriptionId'>
/// Subscription Id.
/// </param>
/// <param name='body'>
/// Subscription Name
/// </param>
/// <param name='customHeaders'>
/// Headers that will be added to request.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
/// <exception cref="ErrorResponseBodyException">
/// Thrown when the operation returned an invalid status code
/// </exception>
/// <exception cref="SerializationException">
/// Thrown when unable to deserialize the response
/// </exception>
/// <exception cref="ValidationException">
/// Thrown when a required parameter is null
/// </exception>
/// <exception cref="System.ArgumentNullException">
/// Thrown when a required parameter is null
/// </exception>
/// <return>
/// A response object containing the response body and response headers.
/// </return>
public async Task <AzureOperationResponse <RenamedSubscriptionId> > RenameWithHttpMessagesAsync(string subscriptionId, SubscriptionName body, Dictionary <string, List <string> > customHeaders = null, CancellationToken cancellationToken = default(CancellationToken))
{
if (subscriptionId == null)
{
throw new ValidationException(ValidationRules.CannotBeNull, "subscriptionId");
}
if (body == null)
{
throw new ValidationException(ValidationRules.CannotBeNull, "body");
}
string apiVersion = "2021-10-01";
// Tracing
bool _shouldTrace = ServiceClientTracing.IsEnabled;
string _invocationId = null;
if (_shouldTrace)
{
_invocationId = ServiceClientTracing.NextInvocationId.ToString();
Dictionary <string, object> tracingParameters = new Dictionary <string, object>();
tracingParameters.Add("subscriptionId", subscriptionId);
tracingParameters.Add("body", body);
tracingParameters.Add("apiVersion", apiVersion);
tracingParameters.Add("cancellationToken", cancellationToken);
ServiceClientTracing.Enter(_invocationId, this, "Rename", tracingParameters);
}
// Construct URL
var _baseUrl = Client.BaseUri.AbsoluteUri;
var _url = new System.Uri(new System.Uri(_baseUrl + (_baseUrl.EndsWith("/") ? "" : "/")), "subscriptions/{subscriptionId}/providers/Microsoft.Subscription/rename").ToString();
_url = _url.Replace("{subscriptionId}", System.Uri.EscapeDataString(subscriptionId));
List <string> _queryParameters = new List <string>();
if (apiVersion != null)
{
_queryParameters.Add(string.Format("api-version={0}", System.Uri.EscapeDataString(apiVersion)));
}
if (_queryParameters.Count > 0)
{
_url += (_url.Contains("?") ? "&" : "?") + string.Join("&", _queryParameters);
}
// Create HTTP transport objects
var _httpRequest = new HttpRequestMessage();
HttpResponseMessage _httpResponse = null;
_httpRequest.Method = new HttpMethod("POST");
_httpRequest.RequestUri = new System.Uri(_url);
// Set Headers
if (Client.GenerateClientRequestId != null && Client.GenerateClientRequestId.Value)
{
_httpRequest.Headers.TryAddWithoutValidation("x-ms-client-request-id", System.Guid.NewGuid().ToString());
}
if (Client.AcceptLanguage != null)
{
if (_httpRequest.Headers.Contains("accept-language"))
{
_httpRequest.Headers.Remove("accept-language");
}
_httpRequest.Headers.TryAddWithoutValidation("accept-language", Client.AcceptLanguage);
}
if (customHeaders != null)
{
foreach (var _header in customHeaders)
{
if (_httpRequest.Headers.Contains(_header.Key))
{
_httpRequest.Headers.Remove(_header.Key);
}
_httpRequest.Headers.TryAddWithoutValidation(_header.Key, _header.Value);
}
}
// Serialize Request
string _requestContent = null;
if (body != null)
{
_requestContent = Rest.Serialization.SafeJsonConvert.SerializeObject(body, Client.SerializationSettings);
_httpRequest.Content = new StringContent(_requestContent, System.Text.Encoding.UTF8);
_httpRequest.Content.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json; charset=utf-8");
}
// Set Credentials
if (Client.Credentials != null)
{
cancellationToken.ThrowIfCancellationRequested();
await Client.Credentials.ProcessHttpRequestAsync(_httpRequest, cancellationToken).ConfigureAwait(false);
}
// Send Request
if (_shouldTrace)
{
ServiceClientTracing.SendRequest(_invocationId, _httpRequest);
}
cancellationToken.ThrowIfCancellationRequested();
_httpResponse = await Client.HttpClient.SendAsync(_httpRequest, cancellationToken).ConfigureAwait(false);
if (_shouldTrace)
{
ServiceClientTracing.ReceiveResponse(_invocationId, _httpResponse);
}
HttpStatusCode _statusCode = _httpResponse.StatusCode;
cancellationToken.ThrowIfCancellationRequested();
string _responseContent = null;
if ((int)_statusCode != 200)
{
var ex = new ErrorResponseBodyException(string.Format("Operation returned an invalid status code '{0}'", _statusCode));
try
{
_responseContent = await _httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
ErrorResponseBody _errorBody = Rest.Serialization.SafeJsonConvert.DeserializeObject <ErrorResponseBody>(_responseContent, Client.DeserializationSettings);
if (_errorBody != null)
{
ex.Body = _errorBody;
}
}
catch (JsonException)
{
// Ignore the exception
}
ex.Request = new HttpRequestMessageWrapper(_httpRequest, _requestContent);
ex.Response = new HttpResponseMessageWrapper(_httpResponse, _responseContent);
if (_shouldTrace)
{
ServiceClientTracing.Error(_invocationId, ex);
}
_httpRequest.Dispose();
if (_httpResponse != null)
{
_httpResponse.Dispose();
}
throw ex;
}
// Create Result
var _result = new AzureOperationResponse <RenamedSubscriptionId>();
_result.Request = _httpRequest;
_result.Response = _httpResponse;
if (_httpResponse.Headers.Contains("x-ms-request-id"))
{
_result.RequestId = _httpResponse.Headers.GetValues("x-ms-request-id").FirstOrDefault();
}
// Deserialize Response
if ((int)_statusCode == 200)
{
_responseContent = await _httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
try
{
_result.Body = Rest.Serialization.SafeJsonConvert.DeserializeObject <RenamedSubscriptionId>(_responseContent, Client.DeserializationSettings);
}
catch (JsonException ex)
{
_httpRequest.Dispose();
if (_httpResponse != null)
{
_httpResponse.Dispose();
}
throw new SerializationException("Unable to deserialize the response.", _responseContent, ex);
}
}
if (_shouldTrace)
{
ServiceClientTracing.Exit(_invocationId, _result);
}
return(_result);
}
/// <summary>
/// The get subscription.
/// </summary>
/// <param name="name">
/// The name.
/// </param>
/// <param name="subscriber">
/// The subscriber.
/// </param>
/// <param name="target">
/// The target.
/// </param>
/// <returns>
/// The <see cref="Subscription"/>.
/// </returns>
private Subscription GetSubscription(SubscriptionName name, User subscriber, User target)
{
Subscription result =
subscriber.UserProfile.Subscriptions.FirstOrDefault(
p => p.Target.Key.Equals(target.Key) && p.Type.Name.Equals(name));
return result;
}
public async Task InitalizeAsync(CancellationToken cancellationToken)
{
PublisherServiceApiClient publisherService = await PublisherServiceApiClient.CreateAsync(cancellationToken);
SubscriberServiceApiClient subscriberService = await SubscriberServiceApiClient.CreateAsync(cancellationToken);
// ensure each topic exists
foreach (string topicId in Topics.AllTopics)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
TopicName topicName = new TopicName(m_projectId, topicId);
try
{
await publisherService.GetTopicAsync(topicName, cancellationToken);
}
catch (RpcException)
{
Topic topic = await publisherService.CreateTopicAsync(topicName, cancellationToken);
m_logger.Info($"Created topic {topic.Name}");
}
m_publisherClients.Add(topicName, await PublisherClient.CreateAsync(topicName));
}
// ensure each subscription exists
foreach (var(topicId, subscriptionId) in Subscriptions.AllSubscriptions)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
SubscriptionName subscriptionName = new SubscriptionName(m_projectId, subscriptionId);
try
{
await subscriberService.GetSubscriptionAsync(subscriptionName, cancellationToken);
}
catch (RpcException)
{
Subscription subscription = await subscriberService.CreateSubscriptionAsync(
new Subscription
{
TopicAsTopicName = new TopicName(m_projectId, topicId),
SubscriptionName = subscriptionName,
AckDeadlineSeconds = 30,
ExpirationPolicy = new ExpirationPolicy
{
Ttl = Duration.FromTimeSpan(TimeSpan.FromDays(365)),
},
},
cancellationToken);
m_logger.Info($"Created subscription {subscription.Name}");
}
m_subscriberClients.Add(subscriptionName, await SubscriberClient.CreateAsync(subscriptionName));
}
}
private async Task RunBulkMessagingImpl(
TopicName topicName, SubscriptionName subscriptionName,
int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount,
TimeSpan?timeouts = null, int?cancelAfterRecvCount = null, TimeSpan?interPublishDelay = null,
TimeSpan?debugOutputPeriod = null, int?publisherChannelCount = null, int?clientCount = null)
{
// Force messages to be at least 4 bytes long, so an int ID can be used.
minMessageSize = Math.Max(4, minMessageSize);
// Create PublisherClient and SubscriberClient
var publisher = await PublisherClient.CreateAsync(topicName,
clientCreationSettings : new PublisherClient.ClientCreationSettings(
clientCount: publisherChannelCount,
publisherServiceApiSettings: timeouts == null ? null : new PublisherServiceApiSettings
{
PublishSettings = CallSettings
.FromRetry(RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromMilliseconds(100),
maxBackoff: TimeSpan.FromSeconds(6),
backoffMultiplier: 1.3,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable)))
.WithTimeout(timeouts.Value)
}
)).ConfigureAwait(false);
var subscriber = await SubscriberClient.CreateAsync(subscriptionName,
clientCreationSettings : new SubscriberClient.ClientCreationSettings(clientCount: clientCount),
settings : new SubscriberClient.Settings
{
AckDeadline = timeouts,
FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null)
}).ConfigureAwait(false);
Console.WriteLine("Topic, Subscription, Publisher and Subscriber all created");
// Subscribe
int recvCount = 0; // Count of received messages
int dupCount = 0; // Count of duplicate messages
long recvSum = 0L; // Sum of bytes of received messages
var recvedIds = new ConcurrentDictionary <int, bool>();
var nackedIds = new HashSet <int>();
Task subTask = subscriber.StartAsync((msg, ct) =>
{
int id = BitConverter.ToInt32(msg.Data.ToArray(), 0);
lock (nackedIds)
{
if (nackedIds.Count < initialNackCount)
{
if (nackedIds.Add(id))
{
// This ID not already nacked
Interlocked.Increment(ref recvCount);
return(Task.FromResult(SubscriberClient.Reply.Nack));
}
}
}
bool wasAdded = recvedIds.TryAdd(id, false);
if (wasAdded)
{
var localRecvCount = Interlocked.Increment(ref recvCount);
Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x));
if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount)
{
// Test finished, so stop subscriber
Console.WriteLine("All msgs received, stopping subscriber.");
Task unused = subscriber.StopAsync(TimeSpan.FromSeconds(15));
}
}
else
{
Interlocked.Add(ref dupCount, 1);
}
// ACK all messages
return(Task.FromResult(SubscriberClient.Reply.Ack));
});
// Publish
var rnd = new Random(1234);
var activePubs = new HashSet <Task>();
int sentCount = 0;
long sentSum = 0L; // Sum of bytes of sent messages
// Watchdog to report progress and fail test on deadlock
CancellationTokenSource watchdogCts = new CancellationTokenSource();
Task.Run(async() =>
{
var debugOutputPeriod1 = debugOutputPeriod ?? TimeSpan.FromSeconds(1);
int prevSentCount = -1;
int prevRecvCount = -1;
int noProgressCount = 0;
while (true)
{
await Task.Delay(debugOutputPeriod1, watchdogCts.Token).ConfigureAwait(false);
var localSentCount = Interlocked.Add(ref sentCount, 0);
var localRecvCount = Interlocked.Add(ref recvCount, 0);
var localDupCount = Interlocked.Add(ref dupCount, 0);
if (prevSentCount == localSentCount && prevRecvCount == localRecvCount)
{
if (noProgressCount > 60)
{
// Deadlock, shutdown subscriber, and cancel
Console.WriteLine("Deadlock detected. Cancelling test");
subscriber.StopAsync(new CancellationToken(true));
watchdogCts.Cancel();
break;
}
noProgressCount += 1;
}
else
{
noProgressCount = 0;
}
prevSentCount = localSentCount;
prevRecvCount = localRecvCount;
Console.WriteLine($"[{DateTime.Now}] Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})");
}
});
for (int i = 0; i < messageCount; i++)
{
if (watchdogCts.IsCancellationRequested)
{
Assert.True(false, "Test cancelled by watchdog");
}
if (subTask.IsCompleted)
{
break;
}
if (i > 0 && interPublishDelay is TimeSpan delay)
{
await Task.Delay(delay, watchdogCts.Token).ConfigureAwait(false);
}
var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1);
var msg = new byte[msgSize];
rnd.NextBytes(msg);
// Insert an int ID into message
Array.Copy(BitConverter.GetBytes(i), msg, 4);
sentSum += msg.Sum(x => (long)x);
// Send message, and record Task
var pubTask = publisher.PublishAsync(msg);
Interlocked.Increment(ref sentCount);
activePubs.Locked(() => activePubs.Add(pubTask));
// Remove Task from active when the message has been sent to server
pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask)));
// If too many messages are currently in flight, wait a bit
while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight)
{
await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
}
}
Console.WriteLine("Publishing complete");
// Wait for all messages to be sent to server
await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false);
Console.WriteLine("Publishing completed sending to server");
// Wait for subscriber to finish shutdown
await subTask.ConfigureAwait(false);
watchdogCts.Cancel();
Console.WriteLine("Subscriber finished shutdown");
Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}");
if (cancelAfterRecvCount is int cancelAfter)
{
Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}");
}
else
{
// Check that all messages are correctly received.
Assert.Equal(messageCount + initialNackCount, recvCount);
// This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator.
Assert.Equal(sentSum, recvSum);
}
Console.WriteLine("Test complete.");
}
public void Subscribe <T, TH>(Func <TH> handler, string topicName, int maxConcurrent = 1)
where T : Event
where TH : IEventHandler <T>
{
new Thread(async() => {
try {
string subscription = _busSettings.SubscriptionName.ToLower() + "-" + topicName.ToLower();
SubscriberClient subscriber;
CreateSubscription(topicName, subscription);
// Pull messages from the subscription using SimpleSubscriber.
SubscriptionName subscriptionName = new SubscriptionName(_busSettings.ProjectId, subscription);
subscriber = await SubscriberClient.CreateAsync(
subscriptionName,
null,
new SubscriberClient.Settings {
FlowControlSettings = new Google.Api.Gax.FlowControlSettings(maxConcurrent, null)
}
);
await subscriber.StartAsync(async(PubsubMessage message, CancellationToken token) => {
T eventMessage;
if ((_busSettings.Token != null && _busSettings.Token != "") && (!message.Attributes.ContainsKey("token") || message.Attributes["token"] != _busSettings.Token))
{
return(SubscriberClient.Reply.Ack);
}
try{
eventMessage = JsonConvert.DeserializeObject <T>(message.Data.ToStringUtf8());
}catch (JsonException ex) {
Console.WriteLine(ex.Message);
return(SubscriberClient.Reply.Ack);
}
try{
eventMessage.EventId = message.MessageId;
eventMessage.Timestamp = message.PublishTime.Seconds * 1000;
}catch (NullReferenceException ex) {
Console.WriteLine(ex.Message);
return(SubscriberClient.Reply.Ack);
}
var invoke = handler.DynamicInvoke();
var concreteType = typeof(IEventHandler <>).MakeGenericType(typeof(T));
EventResult result = await(Task <EventResult>) concreteType.GetMethod("Handle").Invoke(invoke, new object[] { eventMessage, null });
if (result == EventResult.Success)
{
return(SubscriberClient.Reply.Ack);
}
else
{
return(SubscriberClient.Reply.Nack);
}
});
new Thread(() => Subscribe <T, TH>(handler, topicName, maxConcurrent)).Start();
}
// Restart when connection fail
catch (RpcException ex)
{
Console.WriteLine(ex.Message);
new Thread(() => Subscribe <T, TH>(handler, topicName, maxConcurrent)).Start();
return;
}
}).Start();
}
private async Task RunBulkMessaging(
int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount,
TimeSpan?timeouts = null, int?cancelAfterRecvCount = null)
{
// Force messages to be at least 4 bytes long, so an int ID can be used.
minMessageSize = Math.Max(4, minMessageSize);
var topicId = _fixture.CreateTopicId();
var subscriptionId = _fixture.CreateSubscriptionId();
Console.WriteLine("BulkMessaging test");
Console.WriteLine($"{messageCount} messages; of size [{minMessageSize}, {maxMessageSize}]; " +
$"max in-flight: {maxMessagesInFlight}, initialNacks: {initialNackCount}, cancelAfterRecvCount: {cancelAfterRecvCount}");
// Create topic
var topicName = new TopicName(_fixture.ProjectId, topicId);
var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false);
await publisher.CreateTopicAsync(topicName).ConfigureAwait(false);
// Subscribe to the topic
var subscriber = await SubscriberClient.CreateAsync().ConfigureAwait(false);
var subscriptionName = new SubscriptionName(_fixture.ProjectId, subscriptionId);
await subscriber.CreateSubscriptionAsync(subscriptionName, topicName, null, 60).ConfigureAwait(false);
// Create SimplePublisher and SimpleSubscriber
var simplePublisher = await SimplePublisher.CreateAsync(topicName, clientCreationSettings : timeouts == null?null :
new SimplePublisher.ClientCreationSettings(
publisherSettings: new PublisherSettings
{
PublishSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings(
retryBackoff: PublisherSettings.GetMessagingRetryBackoff(),
timeoutBackoff: new BackoffSettings(timeouts.Value, timeouts.Value, 1.0),
totalExpiration: Expiration.FromTimeout(timeouts.Value),
retryFilter: PublisherSettings.NonIdempotentRetryFilter
)))
}
)).ConfigureAwait(false);
var simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName,
settings : new SimpleSubscriber.Settings
{
StreamAckDeadline = timeouts,
FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null)
}).ConfigureAwait(false);
Console.WriteLine("Topic, Subscription, SimplePublisher and SimpleSubscriber all created");
// Subscribe
object recvLock = new object();
int recvCount = 0; // Count of received messages
int dupCount = 0; // Count of duplicate messages
long recvSum = 0L; // Sum of bytes of received messages
var recvedIds = new ConcurrentDictionary <int, bool>();
var nackedIds = new HashSet <int>();
Task subTask = simpleSubscriber.StartAsync((msg, ct) =>
{
int id = BitConverter.ToInt32(msg.Data.ToArray(), 0);
lock (nackedIds)
{
if (nackedIds.Count < initialNackCount)
{
if (nackedIds.Add(id))
{
// This ID not already nacked
Interlocked.Increment(ref recvCount);
return(Task.FromResult(SimpleSubscriber.Reply.Nack));
}
}
}
bool wasAdded = recvedIds.TryAdd(id, false);
if (wasAdded)
{
var localRecvCount = Interlocked.Increment(ref recvCount);
Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x));
if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount)
{
// Test finished, so stop subscriber
Task unused = simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15));
}
}
else
{
Interlocked.Add(ref dupCount, 1);
}
// ACK all messages
return(Task.FromResult(SimpleSubscriber.Reply.Ack));
});
// Publish
var rnd = new Random(1234);
var activePubs = new HashSet <Task>();
int sentCount = 0;
long sentSum = 0L; // Sum of bytes of sent messages
// Watchdog to report progress and fail test on deadlock
CancellationTokenSource watchdogCts = new CancellationTokenSource();
Task.Run(async() =>
{
int prevSentCount = -1;
int prevRecvCount = -1;
int noProgressCount = 0;
while (!watchdogCts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(1), watchdogCts.Token).ConfigureAwait(false);
var localSentCount = Interlocked.Add(ref sentCount, 0);
var localRecvCount = Interlocked.Add(ref recvCount, 0);
var localDupCount = Interlocked.Add(ref dupCount, 0);
if (prevSentCount == localSentCount && prevRecvCount == localRecvCount)
{
if (noProgressCount > 100)
{
// Deadlock, shutdown subscriber, and cancel
Console.WriteLine("Deadlock detected. Cancelling test");
simpleSubscriber.StopAsync(new CancellationToken(true));
watchdogCts.Cancel();
break;
}
noProgressCount += 1;
}
else
{
noProgressCount = 0;
}
prevSentCount = localSentCount;
prevRecvCount = localRecvCount;
Console.WriteLine($"Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})");
}
});
for (int i = 0; i < messageCount; i++)
{
if (watchdogCts.IsCancellationRequested)
{
Assert.True(false, "Test cancelled by watchdog");
}
if (subTask.IsCompleted)
{
break;
}
var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1);
var msg = new byte[msgSize];
rnd.NextBytes(msg);
// Insert an int ID into message
Array.Copy(BitConverter.GetBytes(i), msg, 4);
sentSum += msg.Sum(x => (long)x);
// Send message, and record Task
var pubTask = simplePublisher.PublishAsync(msg);
Interlocked.Increment(ref sentCount);
activePubs.Locked(() => activePubs.Add(pubTask));
// Remove Task from active when the message has been sent to server
pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask)));
// If too many messages are currently in flight, wait a bit
while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight)
{
await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
}
}
Console.WriteLine("Publishing complete");
// Wait for all messages to be sent to server
await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false);
Console.WriteLine("Publishing completed sending to server");
// Wait for subscriber to finish shutdown
await subTask.ConfigureAwait(false);
watchdogCts.Cancel();
Console.WriteLine("Subscriber finished shutdown");
Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}");
if (cancelAfterRecvCount is int cancelAfter)
{
Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}");
}
else
{
// Check that all messages are correctly received.
Assert.Equal(messageCount + initialNackCount, recvCount);
// This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator.
Assert.Equal(sentSum, recvSum);
}
}
public async Task DeleteSubscription(string subscriptionName, CancellationToken cancellationToken = default)
{
var subscription = new SubscriptionName(ProjectId, subscriptionName);
await Subscriber.DeleteSubscriptionAsync(subscription, cancellationToken);
}
public async Task Acknowledge(string subscriptionName, IEnumerable <string> ackIds, CancellationToken cancellationToken = default)
{
var subscription = new SubscriptionName(ProjectId, subscriptionName);
await Subscriber.AcknowledgeAsync(subscription, ackIds, cancellationToken);
}
/// <summary>
/// Renames the subscription
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='body'>
/// Subscription Name
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
public static async Task <RenamedSubscriptionId> RenameAsync(this ISubscriptionsOperations operations, SubscriptionName body, CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.RenameWithHttpMessagesAsync(body, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
/// <summary>
/// Renames the subscription
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='body'>
/// Subscription Name
/// </param>
public static RenamedSubscriptionId Rename(this ISubscriptionsOperations operations, SubscriptionName body)
{
return(operations.RenameAsync(body).GetAwaiter().GetResult());
}
// [END dlp_inspect_datastore]
// [START dlp_inspect_gcs]
public static object InspectGCS(
string projectId,
string minLikelihood,
int maxFindings,
bool includeQuote,
IEnumerable <InfoType> infoTypes,
string bucketName,
string topicId,
string subscriptionId)
{
var inspectJob = new InspectJobConfig
{
StorageConfig = new StorageConfig
{
CloudStorageOptions = new CloudStorageOptions
{
FileSet = new CloudStorageOptions.Types.FileSet {
Url = $"gs://{bucketName}/*.txt"
},
BytesLimitPerFile = 1073741824
},
},
InspectConfig = new InspectConfig
{
InfoTypes = { infoTypes },
ExcludeInfoTypes = false,
IncludeQuote = includeQuote,
Limits = new FindingLimits
{
MaxFindingsPerRequest = maxFindings
},
MinLikelihood = (Likelihood)System.Enum.Parse(typeof(Likelihood), minLikelihood)
},
Actions =
{
new Google.Cloud.Dlp.V2.Action
{
// Send results to Pub/Sub topic
PubSub = new Google.Cloud.Dlp.V2.Action.Types.PublishToPubSub
{
Topic = topicId,
}
}
}
};
// Issue Create Dlp Job Request
DlpServiceClient client = DlpServiceClient.Create();
var request = new CreateDlpJobRequest
{
InspectJob = inspectJob,
ParentAsProjectName = new Google.Cloud.Dlp.V2.ProjectName(projectId),
};
// We need created job name
var dlpJob = client.CreateDlpJob(request);
// Get a pub/sub subscription and listen for DLP results
var fireEvent = new ManualResetEventSlim();
var subscriptionName = new SubscriptionName(projectId, subscriptionId);
var subscriberClient = SubscriberServiceApiClient.Create();
var subscriber = SubscriberClient.Create(subscriptionName, new[] { subscriberClient });
subscriber.StartAsync(
(pubSubMessage, cancellationToken) =>
{
// Given a message that we receive on this subscription, we should either acknowledge or decline it
if (pubSubMessage.Attributes["DlpJobName"] == dlpJob.Name)
{
fireEvent.Set();
return(Task.FromResult(SubscriberClient.Reply.Ack));
}
return(Task.FromResult(SubscriberClient.Reply.Nack));
});
// We block here until receiving a signal from a separate thread that is waiting on a message indicating receiving a result of Dlp job
if (fireEvent.Wait(TimeSpan.FromMinutes(1)))
{
// Stop the thread that is listening to messages as a result of StartAsync call earlier
subscriber.StopAsync(CancellationToken.None).Wait();
// Now we can inspect full job results
var job = client.GetDlpJob(new GetDlpJobRequest {
DlpJobName = new DlpJobName(projectId, dlpJob.Name)
});
// Inspect Job details
Console.WriteLine($"Processed bytes: {job.InspectDetails.Result.ProcessedBytes}");
Console.WriteLine($"Total estimated bytes: {job.InspectDetails.Result.TotalEstimatedBytes}");
var stats = job.InspectDetails.Result.InfoTypeStats;
Console.WriteLine("Found stats:");
foreach (var stat in stats)
{
Console.WriteLine($"{stat.InfoType.Name}");
}
}
else
{
Console.WriteLine("Error: The wait failed on timeout");
}
return(0);
}
// [END dlp_k_anonymity]
// [START dlp_l_diversity]
public static object LDiversity(
string callingProjectId,
string tableProjectId,
string datasetId,
string tableId,
string topicId,
string subscriptionId,
IEnumerable <FieldId> quasiIds,
string sensitiveAttribute)
{
var dlp = DlpServiceClient.Create();
// Construct + submit the job
var ldiversityConfig = new LDiversityConfig
{
SensitiveAttribute = new FieldId {
Name = sensitiveAttribute
},
QuasiIds = { quasiIds }
};
var config = new RiskAnalysisJobConfig
{
PrivacyMetric = new PrivacyMetric
{
LDiversityConfig = ldiversityConfig
},
SourceTable = new BigQueryTable
{
ProjectId = tableProjectId,
DatasetId = datasetId,
TableId = tableId
},
Actions =
{
new Google.Cloud.Dlp.V2.Action
{
PubSub = new PublishToPubSub
{
Topic = $"projects/{callingProjectId}/topics/{topicId}"
}
}
}
};
var submittedJob = dlp.CreateDlpJob(
new CreateDlpJobRequest
{
ParentAsProjectName = new ProjectName(callingProjectId),
RiskJob = config
});
// Listen to pub/sub for the job
var subscriptionName = new SubscriptionName(callingProjectId, subscriptionId);
var subscriber = SubscriberClient.CreateAsync(
subscriptionName).Result;
// SimpleSubscriber runs your message handle function on multiple
// threads to maximize throughput.
var done = new ManualResetEventSlim(false);
subscriber.StartAsync((PubsubMessage message, CancellationToken cancel) =>
{
if (message.Attributes["DlpJobName"] == submittedJob.Name)
{
Thread.Sleep(500); // Wait for DLP API results to become consistent
done.Set();
return(Task.FromResult(SubscriberClient.Reply.Ack));
}
else
{
return(Task.FromResult(SubscriberClient.Reply.Nack));
}
});
done.Wait(TimeSpan.FromMinutes(10)); // 10 minute timeout; may not work for large jobs
subscriber.StopAsync(CancellationToken.None).Wait();
// Process results
var resultJob = dlp.GetDlpJob(
new GetDlpJobRequest
{
DlpJobName = DlpJobName.Parse(submittedJob.Name)
});
var result = resultJob.RiskDetails.LDiversityResult;
for (var bucketIdx = 0; bucketIdx < result.SensitiveValueFrequencyHistogramBuckets.Count; bucketIdx++)
{
var bucket = result.SensitiveValueFrequencyHistogramBuckets[bucketIdx];
Console.WriteLine($"Bucket {bucketIdx}");
Console.WriteLine($" Bucket size range: [{bucket.SensitiveValueFrequencyLowerBound}, {bucket.SensitiveValueFrequencyUpperBound}].");
Console.WriteLine($" {bucket.BucketSize} unique value(s) total.");
foreach (var bucketValue in bucket.BucketValues)
{
// 'UnpackValue(x)' is a prettier version of 'x.toString()'
Console.WriteLine($" Quasi-ID values: [{String.Join(',', bucketValue.QuasiIdsValues.Select(x => DlpSamplesUtils.UnpackValue(x)))}]");
Console.WriteLine($" Class size: {bucketValue.EquivalenceClassSize}");
foreach (var topValue in bucketValue.TopSensitiveValues)
{
Console.WriteLine($" Sensitive value {DlpSamplesUtils.UnpackValue(topValue.Value)} occurs {topValue.Count} time(s).");
}
}
}
return(0);
}
public async Task StreamingPull()
{
string projectId = _fixture.ProjectId;
string topicId = _fixture.CreateTopicId();
string subscriptionId = _fixture.CreateSubscriptionId();
// Snippet: StreamingPull(*, *)
PublisherClient publisher = PublisherClient.Create();
TopicName topicName = new TopicName(projectId, topicId);
publisher.CreateTopic(topicName);
SubscriberClient subscriber = SubscriberClient.Create();
SubscriptionName subscriptionName = new SubscriptionName(projectId, subscriptionId);
subscriber.CreateSubscription(subscriptionName, topicName, null, 60);
// If we don't see all the messages we expect in 10 seconds, we'll cancel the call.
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
CallSettings callSettings = CallSettings.FromCancellationToken(cancellationTokenSource.Token);
SubscriberClient.StreamingPullStream stream = subscriber.StreamingPull(callSettings);
// The first request must include the subscription name and the stream ack deadline
await stream.WriteAsync(new StreamingPullRequest { SubscriptionAsSubscriptionName = subscriptionName, StreamAckDeadlineSeconds = 20 });
Task pullingTask = Task.Run(async() =>
{
int messagesSeen = 0;
IAsyncEnumerator <StreamingPullResponse> responseStream = stream.ResponseStream;
// Handle responses as we see them.
while (await responseStream.MoveNext())
{
StreamingPullResponse response = responseStream.Current;
Console.WriteLine("Received streaming response");
foreach (ReceivedMessage message in response.ReceivedMessages)
{
// Messages can contain any data. We'll assume that we know this
// topic publishes UTF-8-encoded text.
Console.WriteLine($"Message text: {message.Message.Data.ToStringUtf8()}");
}
// Acknowledge the messages we've just seen
await stream.WriteAsync(new StreamingPullRequest {
AckIds = { response.ReceivedMessages.Select(rm => rm.AckId) }
});
// If we've seen all the messages we expect, we can complete the streaming call,
// and our next MoveNext call will return false.
messagesSeen += response.ReceivedMessages.Count;
if (messagesSeen == 3)
{
await stream.WriteCompleteAsync();
}
}
});
publisher.Publish(topicName, new[] { new PubsubMessage {
Data = ByteString.CopyFromUtf8("Message 1")
} });
publisher.Publish(topicName, new[] { new PubsubMessage {
Data = ByteString.CopyFromUtf8("Message 2")
} });
publisher.Publish(topicName, new[] { new PubsubMessage {
Data = ByteString.CopyFromUtf8("Message 3")
} });
await pullingTask;
// End snippet
}
// [START dlp_numerical_stats]
public static object NumericalStats(
string callingProjectId,
string tableProjectId,
string datasetId,
string tableId,
string topicId,
string subscriptionId,
string columnName)
{
var dlp = DlpServiceClient.Create();
// Construct + submit the job
var config = new RiskAnalysisJobConfig
{
PrivacyMetric = new PrivacyMetric
{
NumericalStatsConfig = new NumericalStatsConfig
{
Field = new FieldId {
Name = columnName
}
}
},
SourceTable = new BigQueryTable
{
ProjectId = tableProjectId,
DatasetId = datasetId,
TableId = tableId
},
Actions =
{
new Google.Cloud.Dlp.V2.Action
{
PubSub = new PublishToPubSub
{
Topic = $"projects/{callingProjectId}/topics/{topicId}"
}
}
}
};
var submittedJob = dlp.CreateDlpJob(
new CreateDlpJobRequest
{
ParentAsProjectName = new ProjectName(callingProjectId),
RiskJob = config
});
// Listen to pub/sub for the job
var subscriptionName = new SubscriptionName(callingProjectId, subscriptionId);
var subscriber = SubscriberClient.CreateAsync(
subscriptionName).Result;
// SimpleSubscriber runs your message handle function on multiple
// threads to maximize throughput.
var done = new ManualResetEventSlim(false);
subscriber.StartAsync((PubsubMessage message, CancellationToken cancel) =>
{
if (message.Attributes["DlpJobName"] == submittedJob.Name)
{
Thread.Sleep(500); // Wait for DLP API results to become consistent
done.Set();
return(Task.FromResult(SubscriberClient.Reply.Ack));
}
else
{
return(Task.FromResult(SubscriberClient.Reply.Nack));
}
});
done.Wait(TimeSpan.FromMinutes(10)); // 10 minute timeout; may not work for large jobs
subscriber.StopAsync(CancellationToken.None).Wait();
// Process results
var resultJob = dlp.GetDlpJob(
new GetDlpJobRequest
{
DlpJobName = DlpJobName.Parse(submittedJob.Name)
});
var result = resultJob.RiskDetails.NumericalStatsResult;
// 'UnpackValue(x)' is a prettier version of 'x.toString()'
Console.WriteLine($"Value Range: [{DlpSamplesUtils.UnpackValue(result.MinValue)}, {DlpSamplesUtils.UnpackValue(result.MaxValue)}]");
var lastValue = string.Empty;
for (var quantile = 0; quantile < result.QuantileValues.Count; quantile++)
{
var currentValue = DlpSamplesUtils.UnpackValue(result.QuantileValues[quantile]);
if (lastValue != currentValue)
{
Console.WriteLine($"Value at {quantile + 1}% quantile: {currentValue}");
}
lastValue = currentValue;
}
return(0);
}
private static void Start()
{
SubscriberClient _subscriber;
PublisherClient _publisher;
// Instantiates a client
PublisherServiceApiClient publisherApi = PublisherServiceApiClient.Create();
// Subscribe to the topic.
TopicName pubsubTopicName = new TopicName(projectId, topicName);
SubscriptionName subscriptionName = new SubscriptionName(projectId, subscriptionId);
SubscriberServiceApiClient subscriberApi = SubscriberServiceApiClient.Create();
// Creates the new topic
try
{
Topic topic = publisherApi.CreateTopic(pubsubTopicName);
Console.WriteLine($"Topic {topic.Name} created.");
}
catch (Grpc.Core.RpcException e)
when(e.Status.StatusCode == Grpc.Core.StatusCode.AlreadyExists)
{
Console.WriteLine($"Topic {topicName} already exists.");
}
// Create the new subscription
try
{
subscriberApi.CreateSubscription(subscriptionName, pubsubTopicName, null, 120);
Console.WriteLine($"Subscription {subscriptionName.Kind} created.");
}
catch (Grpc.Core.RpcException e) when(e.Status.StatusCode == Grpc.Core.StatusCode.AlreadyExists)
{
// OK
Console.WriteLine($"Subscription {subscriptionName.Kind} already exists");
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
_subscriber = SubscriberClient.Create(subscriptionName, new[] { subscriberApi });
_publisher = PublisherClient.Create(pubsubTopicName, new[] { publisherApi });
_publisher.PublishAsync("Bla-Bla-Bla-Message.");
_subscriber.StartAsync((message, token) =>
{
string data = message.Data.ToStringUtf8();
try
{
Console.WriteLine($"Pubsub message id={message.MessageId}, " +
$"created at {message.PublishTime}, data{message.Data.ToStringUtf8()}");
// TODO: Replace with ACK
return(System.Threading.Tasks.Task.FromResult(SubscriberClient.Reply.Nack));
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return(System.Threading.Tasks.Task.FromResult(SubscriberClient.Reply.Nack));
}
});
// VARIAN II :
// Pull messages from the subscription.We're returning immediately, whether or not there
// are messages; in other cases you'll want to allow the call to wait until a message arrives.
PullResponse response = subscriberApi.Pull(subscriptionName, returnImmediately: true, maxMessages: 10);
foreach (ReceivedMessage received in response.ReceivedMessages)
{
PubsubMessage msg = received.Message;
Console.WriteLine($"Received message {msg.MessageId} published at {msg.PublishTime.ToDateTime()}");
Console.WriteLine($"Text: '{msg.Data.ToStringUtf8()}'");
}
// Acknowledge that we've received the messages. If we don't do this within 60 seconds (as specified
// when we created the subscription) we'll receive the messages again when we next pull.
subscriberApi.Acknowledge(subscriptionName, response.ReceivedMessages.Select(m => m.AckId));
// Tidy up by deleting the subscription and the topic.
subscriberApi.DeleteSubscription(subscriptionName);
publisherApi.DeleteTopic(pubsubTopicName);
}
// [END dlp_l_diversity]
// [START dlp_k_map]
public static object KMap(
string callingProjectId,
string tableProjectId,
string datasetId,
string tableId,
string topicId,
string subscriptionId,
IEnumerable <FieldId> quasiIds,
IEnumerable <InfoType> infoTypes,
string regionCode)
{
var dlp = DlpServiceClient.Create();
// Construct + submit the job
var kmapEstimationConfig = new KMapEstimationConfig
{
QuasiIds =
{
quasiIds.Zip(
infoTypes,
(Field, InfoType) => new TaggedField
{
Field = Field,
InfoType = InfoType
}
)
},
RegionCode = regionCode
};
var config = new RiskAnalysisJobConfig()
{
PrivacyMetric = new PrivacyMetric
{
KMapEstimationConfig = kmapEstimationConfig
},
SourceTable = new BigQueryTable
{
ProjectId = tableProjectId,
DatasetId = datasetId,
TableId = tableId
},
Actions =
{
new Google.Cloud.Dlp.V2.Action
{
PubSub = new PublishToPubSub
{
Topic = $"projects/{callingProjectId}/topics/{topicId}"
}
}
}
};
var submittedJob = dlp.CreateDlpJob(
new CreateDlpJobRequest
{
ParentAsProjectName = new ProjectName(callingProjectId),
RiskJob = config
});
// Listen to pub/sub for the job
var subscriptionName = new SubscriptionName(
callingProjectId,
subscriptionId);
var subscriber = SubscriberClient.CreateAsync(
subscriptionName).Result;
// SimpleSubscriber runs your message handle function on multiple
// threads to maximize throughput.
var done = new ManualResetEventSlim(false);
subscriber.StartAsync((PubsubMessage message, CancellationToken cancel) =>
{
if (message.Attributes["DlpJobName"] == submittedJob.Name)
{
Thread.Sleep(500); // Wait for DLP API results to become consistent
done.Set();
return(Task.FromResult(SubscriberClient.Reply.Ack));
}
else
{
return(Task.FromResult(SubscriberClient.Reply.Nack));
}
});
done.Wait(TimeSpan.FromMinutes(10)); // 10 minute timeout; may not work for large jobs
subscriber.StopAsync(CancellationToken.None).Wait();
// Process results
var resultJob = dlp.GetDlpJob(new GetDlpJobRequest
{
DlpJobName = DlpJobName.Parse(submittedJob.Name)
});
var result = resultJob.RiskDetails.KMapEstimationResult;
for (var histogramIdx = 0; histogramIdx < result.KMapEstimationHistogram.Count; histogramIdx++)
{
var histogramValue = result.KMapEstimationHistogram[histogramIdx];
Console.WriteLine($"Bucket {histogramIdx}");
Console.WriteLine($" Anonymity range: [{histogramValue.MinAnonymity}, {histogramValue.MaxAnonymity}].");
Console.WriteLine($" Size: {histogramValue.BucketSize}");
foreach (var datapoint in histogramValue.BucketValues)
{
// 'UnpackValue(x)' is a prettier version of 'x.toString()'
Console.WriteLine($" Values: [{String.Join(',', datapoint.QuasiIdsValues.Select(x => DlpSamplesUtils.UnpackValue(x)))}]");
Console.WriteLine($" Estimated k-map anonymity: {datapoint.EstimatedAnonymity}");
}
}
return(0);
}
//private string htmlString;
public PubSubRepository()
{
tn = new TopicName("programmingforthecloudbf", "BFTopic"); //A Queue/Topic will be created to hold the emails to be sent. It will always have the same name DemoTopic, which you can change
sn = new SubscriptionName("programmingforthecloudbf", "BFSubscription"); //A Subscription will be created to hold which messages were read or not. It will always have the same name DemoSubscription, which you can change
}
