public async Task <SendQueueResult> SendToQueueBatch(QueueName queueName, List <ReadOnlyMemory <byte> > dataList)
{
try
{
var connection = await GetConnection();
using (var rabbitMQChannel = connection.CreateModel())
{
var queueResult = rabbitMQChannel.QueueDeclare(queue: queueName.ToString(), durable: true, exclusive: false, autoDelete: false, arguments: null);
IBasicPublishBatch publishBatch = rabbitMQChannel.CreateBasicPublishBatch();
foreach (var data in dataList)
{
publishBatch.Add(exchange: "", routingKey: queueName.ToString(), mandatory: true, null, body: data);
}
publishBatch.Publish();
}
return(new SendQueueResult {
IsSuccess = true, ErrorMessage = ""
});
}
catch (Exception ex)
{
return(new SendQueueResult {
IsSuccess = false, ErrorMessage = ex.Message
});
}
}
/// <summary>
/// Don't Use.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="queueName"></param>
/// <returns></returns>
public async Task <ReceiveQueueResult <T> > ReceiveFromQueue <T>(QueueName queueName)
{
if (IsConnectionIsBlocked())
{
return(new ReceiveQueueResult <T>
{
model = default(T),
IsSuccess = false,
ErrorMessage = $"Connection is blocked. Reason: {connectionBlockedReason}"
});
}
var connection = await GetConnection();
using (var rabbitMQChannel = connection.CreateModel())
{
string consumedMessage = string.Empty;
var queueResult = rabbitMQChannel.QueueDeclare(queue: queueName.ToString(), durable: true, exclusive: false, autoDelete: false, arguments: null);
BasicGetResult result = null;
try
{
result = rabbitMQChannel.BasicGet(queue: queueName.ToString(), true);
if (result != null && result.Body.Length > 0)
{
T model = JsonConvert.DeserializeObject <T>(Encoding.UTF8.GetString(result.Body.ToArray()));
return(new ReceiveQueueResult <T>
{
ErrorMessage = string.Empty,
IsSuccess = true,
model = model
});
}
else
{
return(new ReceiveQueueResult <T> {
IsSuccess = false, ErrorMessage = $"Result is null. QueueName:{queueName}", model = default(T)
});
}
}
catch (Exception ex)
{
return(new ReceiveQueueResult <T> {
IsSuccess = false, ErrorMessage = ex.Message, model = default(T)
});
}
}
}
// Method to put a message on a queue
// Could be expanded to include message attributes, etc., in a SendMessageRequest
public async Task <string> SendMessage(string messageBody)
{
//Console.WriteLine($"Send message to queue\n {Configuration.GetValue<string>("sqsqueue")}");
Console.WriteLine($"Send message to queue\n {Configuration.GetValue<string>("gqueue")}");
var projectId = Configuration.GetValue <string>("projectid");
var location = Configuration.GetValue <string>("location");
var queue = Configuration.GetValue <string>("gqueue");
Console.WriteLine(messageBody);
CloudTasksClient client = CloudTasksClient.Create();
QueueName parent = new QueueName(projectId, location, queue);
var response = client.CreateTask(new CreateTaskRequest
{
Parent = parent.ToString(),
Task = new Google.Cloud.Tasks.V2.Task
{
AppEngineHttpRequest = new AppEngineHttpRequest
{
HttpMethod = HttpMethod.Post,
RelativeUri = "/sendtransaction",
Body = ByteString.CopyFromUtf8(messageBody),
},
ScheduleTime = Timestamp.FromDateTime(
DateTime.UtcNow.AddSeconds(5))
}
});
Console.WriteLine($"Created Task {response.Name}");
return(response.Name);
}
/// <summary>
/// Don't Use.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="queueName"></param>
/// <returns></returns>
public async Task <List <ReceiveQueueResult <T> > > ReceiveBatchFromQueue <T>(QueueName queueName)
{
List <ReceiveQueueResult <T> > receiveQueueResults = new List <ReceiveQueueResult <T> >();
var connection = await GetConnection();
using (var rabbitMQChannel = connection.CreateModel())
{
string consumedMessage;
var consumer = new EventingBasicConsumer(rabbitMQChannel);
consumer.Received += (model, ea) =>
{
var body = ea.Body;
consumedMessage = Encoding.UTF8.GetString(body.ToArray());
receiveQueueResults.Add(new ReceiveQueueResult <T>
{
ErrorMessage = string.Empty,
IsSuccess = true,
model = JsonConvert.DeserializeObject <T>(consumedMessage)
});
};
consumedMessage = rabbitMQChannel.BasicConsume(queue: queueName.ToString(),
autoAck: true,
consumer: consumer);
}
return(receiveQueueResults);
}
public QueueRepository(QueueName queueName)
{
var queueClient = StorageAccountProvider.Instance.CreateCloudQueueClient();
queue = queueClient.GetQueueReference(queueName.ToString());
queue.CreateIfNotExistsAsync();
}
public string CreateTask(
string projectId = "YOUR-PROJECT-ID",
string location = "us-central1",
string queue = "my-queue",
string url = "http://example.com",
string payload = "Hello World!",
int inSeconds = 0)
{
CloudTasksClient client = CloudTasksClient.Create();
QueueName parent = new QueueName(projectId, location, queue);
var response = client.CreateTask(new CreateTaskRequest
{
Parent = parent.ToString(),
Task = new Task
{
HttpRequest = new HttpRequest
{
HttpMethod = HttpMethod.Post,
Url = url,
Body = ByteString.CopyFromUtf8(payload)
},
ScheduleTime = Timestamp.FromDateTime(
DateTime.UtcNow.AddSeconds(inSeconds))
}
});
Console.WriteLine($"Created Task {response.Name}");
return(response.Name);
}
/// <summary>
/// Obtains the first available task from the specified queue, and changes the client's
/// state into TaskReserved. In that state, no new tasks can be reserved until the
/// task's outcome is evident.
/// </summary>
/// <param name="queue"></param>
/// <returns></returns>
public TaskMessage Reserve(QueueName queue)
{
if (State == RedisQueueState.TaskReserved)
{
throw new TaskAlreadyReservedException("Cannot reserve multiple tasks at once.");
}
if (string.IsNullOrEmpty(queue.ToString()))
{
throw new NoQueueSpecifiedException(
"Parameter <queue> is empty or null. Cannot retrieve task for no queue.");
}
if (TypedClient.Lists[queue.NameWhenPending].Count == 0)
{
throw new QueueIsEmptyException("No tasks available in specified queue.");
}
CurrentTask = TypedClient.Lists[queue.NameWhenPending].RemoveStart();
State = RedisQueueState.TaskReserved;
Log.Info("Reserved task from [" + queue.NameWhenPending + "]");
Log.DebugFormat("Task Parameters: {0}", CurrentTask.Parameters);
CacheCurrentTask();
return(CurrentTask);
}
public async Task <ResponseModel> WriteToQueue(QueueName name, string payload)
{
try
{
CloudStorageAccount storageAccount = this.GetCloudStorageAccount(KEY);
var client = storageAccount.CreateCloudQueueClient();
var path = this.GetCloudContainerAPIPath();
// Retrieve a reference to a container.
string queueNameRef = $"{name.ToString().ToLower().Replace("_","-")}";
//lookup actual queue name. If not found, default to the reference name
string queueName = this.AppSettings.QueueNames.TryGetValue(queueNameRef, out string qn) ? qn : queueNameRef;
CloudQueue queue = client.GetQueueReference(queueName);
// Create the queue if it doesn't already exist
await queue.CreateIfNotExistsAsync();
// Create a message and add it to the queue.
CloudQueueMessage message = new CloudQueueMessage(payload);
await queue.AddMessageAsync(message);
return(ResponseModel.Success());
}
catch (Exception ex)
{
await this.LogErrorAsync("WriteToQueue", ex.Message);
return(ResponseModel.Error(ex));
}
}
public string Name(QueueName name)
{
var val = _ht[name];
if (val == null)
{
val = name.ToString("G");
_ht[name] = val;
}
return val.ToString();
}
/// <summary>
/// Returns all succeeded tasks.
/// </summary>
/// <param name="queue"></param>
/// <returns></returns>
public IList <TaskMessage> SucceededTasks(QueueName queue)
{
if (string.IsNullOrEmpty(queue.ToString()))
{
throw new NoQueueSpecifiedException(
"Parameter <queue> is empty or null. Cannot retrieve task for no queue.");
}
return(TypedClient.Lists[queue.NameWhenSucceeded]);
}
public static Event ToEvent(this BrokeredMessage message,
QueueName queueName)
{
return
new Event()
{
Body = message.GetBody<string>(),
ContentType = message.ContentType,
EventType = queueName.TopicName,
QueueName = queueName.ToString(),
UnderlyingMessage = message
};
}
public static Event ToEvent(this BrokeredMessage message,
QueueName queueName)
{
return
(new Event()
{
Body = message.GetBody <string>(),
ContentType = message.ContentType,
EventType = queueName.TopicName,
QueueName = queueName.ToString(),
UnderlyingMessage = message
});
}
private static string MapToCollectionName(QueueName queueName)
{
var collectionName = queueName.ToString()
.Replace(" ", "_")
.Replace("$", "_");
const string reservedNamespace = "system.";
if (collectionName.StartsWith(reservedNamespace, StringComparison.OrdinalIgnoreCase))
{
collectionName = collectionName.Substring(reservedNamespace.Length);
}
return(collectionName);
}
public static Event ToEvent(this Message message,
QueueName queueName)
{
return
(new Event()
{
Body = Encoding.UTF8.GetString(message.Body),
ContentType = message.ContentType,
EventType = queueName.TopicName,
QueueName = queueName.ToString(),
UnderlyingMessage = message,
Timestamp = message.SystemProperties.EnqueuedTimeUtc
});
}
public async Task <SendQueueResult> SendToQueue(QueueName queueName, ReadOnlyMemory <byte> data)
{
try
{
var connection = await GetConnection();
using (var rabbitMQChannel = connection.CreateModel())
{
var queueResult = rabbitMQChannel.QueueDeclare(queue: queueName.ToString(), durable: true, exclusive: false, autoDelete: false, arguments: null);
rabbitMQChannel.BasicPublish(exchange: "", routingKey: queueName.ToString(), basicProperties: null, body: data);
}
return(new SendQueueResult {
IsSuccess = true, ErrorMessage = ""
});
}
catch (Exception ex)
{
return(new SendQueueResult {
IsSuccess = false, ErrorMessage = ex.Message
});
}
}
public Task <PollerResult <Event> > NextAsync(QueueName name)
{
var q = GetQueue(name);
return(q.Subscriptions[name.SubscriptionName].NextAsync(name)
.ContinueWith((task) =>
{
if (!task.IsFaulted && task.Result.IsSuccessful)
{
task.Result.PollingResult.QueueName = name.ToString();
}
else
{
}
return task.Result;
}));
}
public QueueClient GetQueueClient(QueueName queueName)
{
Func <object> factory = () =>
new QueueClient(_connectionString,
queueName.TopicName);
if (_cache)
{
var lazy =
_cachedClients.GetOrAdd(
queueName.ToString(),
(name) =>
new Lazy <object>(factory));
return((QueueClient)lazy.Value);
}
return((QueueClient)factory());
}
public SubscriptionClient GetSubscriptionClient(QueueName queueName)
{
Func <object> factory = () =>
SubscriptionClient.CreateFromConnectionString(_connectionString,
queueName.TopicName,
queueName.SubscriptionName);
if (_cache)
{
var lazy =
_cachedClients.GetOrAdd(
queueName.ToString(),
(name) =>
new Lazy <object>(factory));
return((SubscriptionClient)lazy.Value);
}
return((SubscriptionClient)factory());
}
async public void CreateTask(CreateContentScanTaskDto scanTaskDto)
{
if (!serverConfiguration.DoSendContentScanningRequest)
{
return;
}
QueueName parent = new QueueName(googleTasksConfiguration.ProjectId, googleTasksConfiguration.Location, googleTasksConfiguration.QueueName);
string payload = JsonSerializer.Serialize(scanTaskDto);
await cloudTasksClient.CreateTaskAsync(new CreateTaskRequest
{
Parent = parent.ToString(),
Task = new Task
{
HttpRequest = new HttpRequest
{
HttpMethod = HttpMethod.Post,
Url = googleTasksConfiguration.GoogleFunctionUrl,
Body = ByteString.CopyFromUtf8(payload)
}
}
});
}
// [START cloud_tasks_appengine_create_task]
public static object CreateTask(
string projectId,
string location,
string queue,
string payload,
int inSeconds)
{
CloudTasksClient client = CloudTasksClient.Create();
QueueName parent = new QueueName(projectId, location, queue);
var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var response = client.CreateTask(new CreateTaskRequest
{
Parent = parent.ToString(),
Task = new Task
{
AppEngineHttpRequest = new AppEngineHttpRequest
{
HttpMethod = HttpMethod.Post,
RelativeUri = "/log_payload",
Body = ByteString.CopyFromUtf8(payload)
},
ScheduleTime = new Timestamp
{
Seconds = (long)(DateTime.Now.AddSeconds(inSeconds) - unixEpoch).TotalSeconds,
Nanos = 0
}
}
});
Console.WriteLine($"Created Task {response.Name}");
return(0);
}
public async Task <uint> GetQueueCount(QueueName queueName)
{
try
{
if (IsConnectionIsBlocked())
{
logger.LogError(new EventId((int)EventName.RabbitMQService, $"{nameof(EventName.RabbitMQService)}/{nameof(GetQueueCount)}"), $"RabbitMQ Connection is blocked. Reason: {connectionBlockedReason}");
return(0);
}
var connection = await GetConnection();
using (var rabbitMQChannel = connection.CreateModel())
{
var queueResult = rabbitMQChannel.QueueDeclare(queue: queueName.ToString(), durable: true, exclusive: false, autoDelete: false, arguments: null);
return(queueResult.MessageCount);
}
}
catch (Exception ex)
{
logger.LogError(new EventId((int)EventName.RabbitMQService, $"{nameof(EventName.RabbitMQService)}/{nameof(GetQueueCount)}"), ex, $"GetQueueCount Error: RabbitMQ Connection is blocked. Reason: {connectionBlockedReason}");
throw;
}
}
/// <summary>
/// Stages indexes. Should run only using one process.
/// </summary>
/// <param name="scope"></param>
/// <param name="documentType"></param>
/// <param name="rebuild"></param>
/// <exception cref="System.ArgumentNullException">scope</exception>
public void Prepare(string scope, string documentType = "", bool rebuild = false)
{
if (String.IsNullOrEmpty(scope))
{
throw new ArgumentNullException("scope");
}
foreach (var builder in _indexBuilders)
{
// skip not requested indexers or index using all if index is not specified
if (!String.IsNullOrEmpty(documentType) && !documentType.Equals(builder.DocumentType))
{
continue;
}
// Execute builder, which will create partitions and put them in the queue
var queueName = new QueueName("index-{0}-{1}-in", scope, builder.DocumentType);
var config = GetBuildConfig(_repository, queueName.Scope, queueName.DocumentType);
var lastBuild = DateTime.UtcNow;
var newBuildDate = lastBuild;
if (config.Status == BuildStatus.NeverStarted.GetHashCode() || rebuild) // build was never started, so set min date
{
rebuild = true;
lastBuild = DateTime.MinValue;
config.LastBuildDate = DateTime.UtcNow.AddYears(-30); // have to set the date to something repository won't complain
}
else
{
lastBuild = config.LastBuildDate.AddSeconds(-30); // make sure we get all the changes
}
// Delete all the records
if (rebuild)
{
_searchProvider.RemoveAll(queueName.Scope, queueName.DocumentType);
}
var partitions = builder.CreatePartitions(queueName.Scope, lastBuild);
var newPartitionsExist = false; // tells if there are any partitions that has been processed
foreach (var partition in partitions)
{
newPartitionsExist = true;
//_observer.Notify(new ConsumeBegin(msg, consumer, envelope.QueueName));
_messageSender.Send(queueName.ToString(), partition);
}
var newBuildStatus = BuildStatus.Started;
if (newPartitionsExist)
{
_messageSender.Send(queueName.ToString(), new SearchIndexStatusMessage(queueName.Scope, queueName.DocumentType, BuildStatus.Completed));
}
else
{
newBuildStatus = BuildStatus.Completed;
}
config.LastBuildDate = newBuildDate;
config.Status = newBuildStatus.GetHashCode();
_repository.UnitOfWork.Commit();
}
}
public static Event ToEvent(this Message message,
QueueName queueName)
{
return(ToEvent(message, queueName.TopicName, queueName.ToString()));
}
public MSMessageQueueHelper(QueueName queueName)
{
QueuePath += queueName.ToString();
}
/// <summary>
/// Stages indexes. Should run only using one process.
/// </summary>
/// <param name="scope"></param>
/// <param name="documentType"></param>
/// <param name="rebuild"></param>
/// <exception cref="System.ArgumentNullException">scope</exception>
public void Prepare(string scope, string documentType = "", bool rebuild = false)
{
if (String.IsNullOrEmpty(scope))
throw new ArgumentNullException("scope");
foreach (var builder in _indexBuilders)
{
// skip not requested indexers or index using all if index is not specified
if (!String.IsNullOrEmpty(documentType) && !documentType.Equals(builder.DocumentType))
continue;
// Execute builder, which will create partitions and put them in the queue
var queueName = new QueueName("index-{0}-{1}-in", scope, builder.DocumentType);
var config = GetBuildConfig(_repository, queueName.Scope, queueName.DocumentType);
var lastBuild = DateTime.UtcNow;
var newBuildDate = lastBuild;
if (config.Status == BuildStatus.NeverStarted.GetHashCode() || rebuild) // build was never started, so set min date
{
rebuild = true;
lastBuild = DateTime.MinValue;
config.LastBuildDate = DateTime.UtcNow.AddYears(-30); // have to set the date to something repository won't complain
}
else
{
lastBuild = config.LastBuildDate.AddSeconds(-30); // make sure we get all the changes
}
// Delete all the records
if (rebuild)
{
_searchProvider.RemoveAll(queueName.Scope, queueName.DocumentType);
}
var partitions = builder.CreatePartitions(queueName.Scope, lastBuild);
var newPartitionsExist = false; // tells if there are any partitions that has been processed
foreach (var partition in partitions)
{
newPartitionsExist = true;
//_observer.Notify(new ConsumeBegin(msg, consumer, envelope.QueueName));
_messageSender.Send(queueName.ToString(), partition);
}
var newBuildStatus = BuildStatus.Started;
if (newPartitionsExist)
{
_messageSender.Send(queueName.ToString(), new SearchIndexStatusMessage(queueName.Scope, queueName.DocumentType, BuildStatus.Completed));
}
else
{
newBuildStatus = BuildStatus.Completed;
}
config.LastBuildDate = newBuildDate;
config.Status = newBuildStatus.GetHashCode();
_repository.UnitOfWork.Commit();
}
}
public Task <PollerResult <Event> > NextAsync(QueueName name)
{
return(NextAsync(name.ToString()));
}
/// <summary>
/// Obtains the first available task from the specified queue, and changes the client's
/// state into TaskReserved. In that state, no new tasks can be reserved until the
/// task's outcome is evident.
/// </summary>
/// <param name="queue"></param>
/// <returns></returns>
public TaskMessage Reserve(QueueName queue)
{
if (State == RedisQueueState.TaskReserved)
throw new TaskAlreadyReservedException("Cannot reserve multiple tasks at once.");
if (string.IsNullOrEmpty(queue.ToString()))
throw new NoQueueSpecifiedException(
"Parameter <queue> is empty or null. Cannot retrieve task for no queue.");
if (TypedClient.Lists[queue.NameWhenPending].Count == 0)
throw new QueueIsEmptyException("No tasks available in specified queue.");
CurrentTask = TypedClient.Lists[queue.NameWhenPending].RemoveStart();
State = RedisQueueState.TaskReserved;
Log.Info("Reserved task from [" + queue.NameWhenPending + "]");
Log.DebugFormat("Task Parameters: {0}", CurrentTask.Parameters);
CacheCurrentTask();
return CurrentTask;
}
public QueueClient GetQueueClient(QueueName queueName)
{
Func<object> factory = () =>
QueueClient.CreateFromConnectionString(_connectionString,
queueName.TopicName);
if (_cache)
{
var lazy =
_cachedClients.GetOrAdd(
queueName.ToString(),
(name) =>
new Lazy<object>(factory));
return (QueueClient)lazy.Value;
}
return (QueueClient)factory();
}
/// <summary>
/// Returns all succeeded tasks.
/// </summary>
/// <param name="queue"></param>
/// <returns></returns>
public IList<TaskMessage> SucceededTasks(QueueName queue)
{
if (string.IsNullOrEmpty(queue.ToString()))
throw new NoQueueSpecifiedException(
"Parameter <queue> is empty or null. Cannot retrieve task for no queue.");
return TypedClient.Lists[queue.NameWhenSucceeded];
}
