public static async Task ServiceBusQueueProcessorAsync(
[ServiceBusTrigger("%SERVICE_BUS_QUEUE%")] Message message, MessageReceiver messageReceiver, ILogger logger)
{
var timestamp = DateTime.UtcNow;
var queueName = Environment.GetEnvironmentVariable("SERVICE_BUS_QUEUE", EnvironmentVariableTarget.Process);
logger.LogTrace($@"[{message.UserProperties[@"TaskId"]}]: Message received at {timestamp}: {JObject.FromObject(message)}");
AppInsightsLogger appInsightsLogger = new AppInsightsLogger(logger, LOGGING_SERVICE_NAME + ": " + queueName, LOGGING_SERVICE_VERSION);
var enqueuedTime = message.ScheduledEnqueueTimeUtc;
var elapsedTimeMs = (timestamp - enqueuedTime).TotalMilliseconds;
var taskId = message.UserProperties["TaskId"].ToString();
var backendUri = message.UserProperties["Uri"].ToString();
var messageBody = Encoding.UTF8.GetString(message.Body);
try
{
appInsightsLogger.LogInformation($"Sending request to {backendUri} for taskId {taskId} from queue {queueName}. Queued for {elapsedTimeMs/60} seconds.", backendUri, taskId);
var client = new HttpClient();
client.DefaultRequestHeaders.Add("taskId", taskId);
var httpContent = new StringContent(messageBody, Encoding.UTF8, "application/json");
var res = await client.PostAsync(backendUri, httpContent);
if (res.StatusCode == (System.Net.HttpStatusCode) 429) // Special return value indicating that the service is busy.
{
var retryDelay = int.Parse(Environment.GetEnvironmentVariable(QUEUE_RETRY_DELAY_MS_VARIABLE_NAME, EnvironmentVariableTarget.Process));
appInsightsLogger.LogInformation($"Service is busy. Will try again in {retryDelay/1000} seconds.", backendUri, taskId);
await UpdateTaskStatus(taskId, backendUri, messageBody, $"Awaiting service availability. Queued for {elapsedTimeMs/60} seconds.", "created", appInsightsLogger);
// Artifical delay is needed since the ServiceBusTrigger will retry immediately.
await Task.Delay(retryDelay);
await messageReceiver.AbandonAsync(message.SystemProperties.LockToken);
throw new ApplicationException($"Service is busy. Will try again in {retryDelay/1000} seconds.");
}
else if (!res.IsSuccessStatusCode)
{
await messageReceiver.CompleteAsync(message.SystemProperties.LockToken); //Need to complete even though we have failure. This removes it from the queue to avoid an infinite state.
appInsightsLogger.LogWarning($"Unable to send request to backend. Status: {res.StatusCode.ToString()}, Reason: {res.ReasonPhrase}", backendUri, taskId);
await UpdateTaskStatus(taskId, backendUri, messageBody, $"Unable to send request to backend.", "failed", appInsightsLogger);
}
else
{
await messageReceiver.CompleteAsync(message.SystemProperties.LockToken);
appInsightsLogger.LogInformation($"taskId {taskId} has successfully been pushed to the backend from queue {queueName}. Queue time: {elapsedTimeMs/60} seconds.", backendUri, taskId);
}
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex, backendUri, taskId);
}
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req, ILogger logger)
{
IDatabase db = null;
AppInsightsLogger appInsightsLogger = new AppInsightsLogger(logger, LOGGING_SERVICE_NAME, LOGGING_SERVICE_VERSION);
string cluster = string.Empty;
string path = string.Empty;
var apiParams = req.GetQueryParameterDictionary();
if (apiParams != null && apiParams.Keys.Contains(SERVICE_CLUSTER_KEY_NAME) && apiParams.Keys.Contains(API_PATH_KEY_NAME))
{
cluster = apiParams[SERVICE_CLUSTER_KEY_NAME];
path = apiParams[API_PATH_KEY_NAME];
}
else
{
return(new BadRequestObjectResult("The cluster and path parameters are requried."));
}
try
{
db = RedisConnection.GetDatabase();
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex, cluster + "/" + path);
return(new StatusCodeResult(500));
}
RedisValue storedCount = RedisValue.Null;
try
{
storedCount = await db.StringGetAsync(APP_INSIGHTS_REQUESTS_KEY_NAME + cluster + "/" + path);
if (storedCount.HasValue)
{
appInsightsLogger.LogInformation("Found status in cache", cluster + "/" + path);
return(new OkObjectResult(storedCount.ToString()));
}
else
{
appInsightsLogger.LogInformation("Found status in cache", cluster + "/" + path);
return(new StatusCodeResult(204));
}
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex, cluster + "/" + path);
return(new StatusCodeResult(500));
}
}
public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req, ILogger logger)
{
IDatabase db = null;
AppInsightsLogger appInsightsLogger = new AppInsightsLogger(logger, LOGGING_SERVICE_NAME, LOGGING_SERVICE_VERSION);
string uuid = "nil";
var apiParams = req.GetQueryParameterDictionary();
if (apiParams != null && apiParams.Keys.Contains(UUID_KEYNAME))
{
uuid = apiParams[UUID_KEYNAME];
appInsightsLogger.LogInformation("Getting status for taskId: " + uuid, URL, uuid);
}
else
{
appInsightsLogger.LogWarning("Called without a taskId", URL);
return(new BadRequestObjectResult("The taskId parameter is requried."));
}
try
{
db = RedisConnection.GetDatabase();
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex, URL, uuid);
return(new StatusCodeResult(500));
}
RedisValue storedStatus = RedisValue.Null;
try
{
storedStatus = db.StringGet(uuid);
if (storedStatus.HasValue)
{
appInsightsLogger.LogInformation("Found status in cache", URL, uuid);
return(new OkObjectResult(storedStatus.ToString()));
}
else
{
appInsightsLogger.LogWarning("Cannot find status in cache", URL, uuid);
return(new StatusCodeResult(204));
}
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex, URL, uuid);
return(new StatusCodeResult(500));
}
}
public void Test_Telemetry_LogInfoWithObject()
{
// Assert
AssertExtensions.DoesNotThrow(() =>
{
// Arrange
var logger = new AppInsightsLogger("test", LogLevel.Debug);
// Act
logger.LogInformation("test", new Test
{
PropA = "propA",
PropB = 1,
PropC = true,
PropD = new SubItem
{
PropE = "propE",
PropF = new List <int> {
1, 2, 3
}
}
});
logger.Flush();
});
}
private static async Task <bool> PublishServiceBusQueueEvent(APITask task, string taskBody, AppInsightsLogger appInsightsLogger)
{
string serviceBusConnectionString = Environment.GetEnvironmentVariable(SERVICE_BUS_CONNECTION_STRING_VARIABLE_NAME, EnvironmentVariableTarget.Process);
// A queue must be created for each endpoint, ex:
// The queue name for http://52.224.89.22/v1/paws/consolidate should be http52.224.89.22v1pawsconsolidate
var queueName = task.Endpoint.Replace(".", string.Empty);
queueName = queueName.Replace("/", string.Empty);
queueName = queueName.Replace(":", string.Empty);
var messageProperties = new Dictionary <string, object>
{
{ "TaskId", task.TaskId },
{ "Uri", task.Endpoint }
};
IQueueClient queueClient = new QueueClient(serviceBusConnectionString, queueName);
try
{
Message message = new Message(Encoding.UTF8.GetBytes(taskBody));
message.UserProperties["TaskId"] = task.TaskId;
message.UserProperties["Uri"] = task.Endpoint;
// Write the body of the message to the console.
Console.WriteLine($"Sending taskId {task.TaskId} to queue {queueName}");
// Send the message to the queue.
await queueClient.SendAsync(message);
await queueClient.CloseAsync();
appInsightsLogger.LogInformation($"Sent task {task.TaskId} to queue {queueName}.", task.Endpoint, task.TaskId);
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex, task.Endpoint, task.TaskId);
return(false);
}
return(true);
}
public void Test_Telemetry_LogInformationMessage()
{
// Assert
AssertExtensions.DoesNotThrow(() =>
{
// Arrange
var logger = new AppInsightsLogger("test", LogLevel.Trace);
Dictionary <string, string> statistics = new Dictionary <string, string>()
{
{ "SomeProp1", "sample1" },
{ "SomeProp2", "sample2" },
{ "SomeProp3", "sample3" }
};
logger.LogInformation("test title", statistics);
logger.Log(LogLevel.Information, new EventId(1, null), "A", null, (a, b) =>
{
return(string.Format("{0}:{1}", a, b));
});
logger.Flush();
});
}
public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req, ILogger logger)
{
IDatabase db = null;
AppInsightsLogger appInsightsLogger = new AppInsightsLogger(logger, LOGGING_SERVICE_NAME, LOGGING_SERVICE_VERSION);
var redisOperation = "insert";
APITask task = null;
if (req.Body != null)
{
string body = string.Empty;
try
{
using (StreamReader reader = new StreamReader(req.Body))
{
if (reader.BaseStream.Length > 0)
{
body = reader.ReadToEnd();
logger.LogInformation("body: " + body);
try
{
appInsightsLogger.LogInformation("DeserializeObject<APITask>(body)");
task = JsonConvert.DeserializeObject <APITask>(body);
}
catch
{
appInsightsLogger.LogInformation("DeserializeObject<APITask>(body[])");
task = JsonConvert.DeserializeObject <APITask[]>(body)[0];
}
appInsightsLogger.LogInformation("task.PublishToGrid: " + task.PublishToGrid.ToString(), task.Endpoint, task.TaskId);
}
else
{
appInsightsLogger.LogWarning("Parameters missing. Unable to create task.");
return(new BadRequestResult());
}
}
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex);
appInsightsLogger.LogRedisUpsert("Redis upsert failed.", redisOperation, task.Timestamp, body);
return(new StatusCodeResult(500));
}
}
else
{
appInsightsLogger.LogWarning("Parameters missing. Unable to create task.");
return(new BadRequestResult());
}
if (!String.IsNullOrWhiteSpace(task.TaskId))
{
appInsightsLogger.LogInformation("Updating status", task.Endpoint, task.TaskId);
redisOperation = "update";
}
else
{
task.TaskId = Guid.NewGuid().ToString();
appInsightsLogger.LogInformation("Generated new taskId: " + task.TaskId, task.Endpoint, task.TaskId);
}
task.Timestamp = DateTime.UtcNow.ToString();
try
{
db = RedisConnection.GetDatabase();
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex, task.Endpoint, task.TaskId);
appInsightsLogger.LogRedisUpsert("Redis upsert failed.", redisOperation, task.Timestamp, task.Endpoint, task.TaskId);
return(new StatusCodeResult(500));
}
string serializedTask = string.Empty;
try
{
var taskBody = task.Body;
task.Body = null;
serializedTask = JsonConvert.SerializeObject(task);
RedisValue res = RedisValue.Null;
appInsightsLogger.LogInformation("Setting Redis task record", task.Endpoint, task.TaskId);
var upsertTransaction = db.CreateTransaction();
upsertTransaction.StringSetAsync(task.TaskId, serializedTask);
// Get seconds since epoch
TimeSpan ts = (DateTime.UtcNow - new DateTime(1970, 1, 1));
int timestamp = (int)ts.TotalSeconds;
appInsightsLogger.LogInformation(string.Format("Adding backend status '{0}' for endpoint.", task.BackendStatus), task.Endpoint, task.TaskId);
upsertTransaction.SortedSetAddAsync(string.Format("{0}_{1}", task.EndpointPath, task.BackendStatus), new SortedSetEntry[] { new SortedSetEntry(task.TaskId, timestamp) });
if (task.BackendStatus.Equals(BACKEND_STATUS_RUNNING))
{
upsertTransaction.SortedSetRemoveAsync(string.Format("{0}_{1}", task.EndpointPath, BACKEND_STATUS_CREATED), task.TaskId);
}
else if (task.BackendStatus.Equals(BACKEND_STATUS_COMPLETED) || task.BackendStatus.Equals(BACKEND_STATUS_FAILED))
{
upsertTransaction.SortedSetRemoveAsync(string.Format("{0}_{1}", task.EndpointPath, BACKEND_STATUS_RUNNING), task.TaskId);
}
bool isSubsequentPipelineCall = false;
bool isPublish = false;
bool.TryParse(task.PublishToGrid.ToString(), out isPublish);
if (isPublish == true || task.PublishToGrid == true)
{
if (string.IsNullOrEmpty(taskBody))
{
appInsightsLogger.LogInformation("It IS a pipeline call", task.Endpoint, task.TaskId);
// This is a subsequent pipeline publish.
isSubsequentPipelineCall = true;
}
else
{
appInsightsLogger.LogInformation("Adding body to redis: " + taskBody, task.Endpoint, task.TaskId);
upsertTransaction.StringSetAsync(string.Format("{0}_{1}", task.TaskId, GRID_PUBLISH_RECORD_KEY), taskBody);
}
}
ExecuteTransaction(upsertTransaction, task, appInsightsLogger);
if (isSubsequentPipelineCall)
{
// We have to get the original body, since it's currently empty.
taskBody = db.StringGet(string.Format("{0}_{1}", task.TaskId, GRID_PUBLISH_RECORD_KEY));
appInsightsLogger.LogInformation("Stored body: " + taskBody, task.Endpoint, task.TaskId);
}
if (task.PublishToGrid)
{
if (PublishEvent(task, taskBody, appInsightsLogger) == false)
{
// Move task to failed
var updateTransaction = db.CreateTransaction();
task.Status = "Failed - unable to send to backend service.";
task.BackendStatus = BACKEND_STATUS_FAILED;
string updateBody = JsonConvert.SerializeObject(task);
updateTransaction.StringSetAsync(task.TaskId, updateBody);
updateTransaction.SortedSetAddAsync(string.Format("{0}_{1}", task.EndpointPath, task.BackendStatus), new SortedSetEntry[] { new SortedSetEntry(task.TaskId, timestamp) });
updateTransaction.SortedSetRemoveAsync(string.Format("{0}_{1}", task.EndpointPath, BACKEND_STATUS_CREATED), task.TaskId);
ExecuteTransaction(updateTransaction, task, appInsightsLogger);
}
}
//LogSetCount(string.Format("{0}_{1}", task.EndpointPath, task.BackendStatus), task, db, appInsightsLogger);
appInsightsLogger.LogRedisUpsert("Redis upsert successful.", redisOperation, task.Timestamp, serializedTask, task.Endpoint, task.TaskId);
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex, task.Endpoint, task.TaskId);
appInsightsLogger.LogRedisUpsert("Redis upsert failed.", redisOperation, task.Timestamp, serializedTask, task.Endpoint, task.TaskId);
return(new StatusCodeResult(500));
}
return(new OkObjectResult(serializedTask));
}
public static async Task <IActionResult> Run([HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req, ILogger logger)
{
AppInsightsLogger appInsightsLogger = new AppInsightsLogger(logger, LOGGING_SERVICE_NAME, LOGGING_SERVICE_VERSION);
string response = string.Empty;
const string SubscriptionValidationEvent = "Microsoft.EventGrid.SubscriptionValidationEvent";
string requestContent = new StreamReader(req.Body).ReadToEnd();
EventGridEvent[] eventGridEvents = JsonConvert.DeserializeObject <EventGridEvent[]>(requestContent);
// We should only have 1 event
foreach (EventGridEvent eventGridEvent in eventGridEvents)
{
JObject dataObject = eventGridEvent.Data as JObject;
// Deserialize the event data into the appropriate type based on event type
if (string.Equals(eventGridEvent.EventType, SubscriptionValidationEvent, StringComparison.OrdinalIgnoreCase))
{
var eventData = dataObject.ToObject <SubscriptionValidationEventData>();
appInsightsLogger.LogInformation($"Got SubscriptionValidation event data, validation code: {eventData.ValidationCode}, topic: {eventGridEvent.Topic}", string.Empty);
// Do any additional validation (as required) and then return back the below response
var responseData = new SubscriptionValidationResponse();
responseData.ValidationResponse = eventData.ValidationCode;
return(new OkObjectResult(responseData));
}
else
{
var backendUri = new Uri(eventGridEvent.Subject);
var client = new HttpClient();
var stringContent = new StringContent(eventGridEvent.Data.ToString(), Encoding.UTF8, "application/json");
try
{
appInsightsLogger.LogInformation($"Sending request to {backendUri} for taskId {eventGridEvent.Id}.", eventGridEvent.Subject, eventGridEvent.Id);
client.DefaultRequestHeaders.Add("taskId", eventGridEvent.Id);
var res = await client.PostAsync(backendUri, stringContent);
if (res.StatusCode == (System.Net.HttpStatusCode) 429) // Special return value indicating that the service is busy. Let event grid handle the retries.
{
appInsightsLogger.LogInformation("Backend service is busy. Event grid will retry with backoff.", eventGridEvent.Subject, eventGridEvent.Id);
}
else if (!res.IsSuccessStatusCode)
{
appInsightsLogger.LogWarning($"Unable to send request to backend. Status: {res.StatusCode.ToString()}, Reason: {res.ReasonPhrase}", eventGridEvent.Subject, eventGridEvent.Id);
}
appInsightsLogger.LogInformation("Request has successfully been pushed to the backend.", eventGridEvent.Subject, eventGridEvent.Id);
return(new StatusCodeResult((int)(res.StatusCode)));
}
catch (Exception ex)
{
appInsightsLogger.LogError(ex, eventGridEvent.Subject, eventGridEvent.Id);
return(new StatusCodeResult(500));
}
}
}
return(new OkResult());
}