public static async Task ProcessSecurityAsync(
[QueueTrigger(OperationConstants.SecurityQueueName, Connection = "StorageConnectionString")] SecurityDetails data,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(Alert),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <Alert> securityAlertRepository,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(SecureScore),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <SecureScore> secureScoreRepository,
[SecureScore(
ApplicationId = "{AppEndpoint.ApplicationId}",
CustomerId = "{Customer.Id}",
KeyVaultEndpoint = "KeyVaultEndpoint",
Period = 1,
Resource = "{AppEndpoint.ServiceAddress}",
SecretName = "{AppEndpoint.ApplicationSecretId}")] List <SecureScore> scores,
[SecurityAlerts(
ApplicationId = "{AppEndpoint.ApplicationId}",
CustomerId = "{Customer.Id}",
KeyVaultEndpoint = "KeyVaultEndpoint",
Resource = "{AppEndpoint.ServiceAddress}",
SecretName = "{AppEndpoint.ApplicationSecretId}")] List <Alert> alerts,
TraceWriter log)
{
log.Info($"Attempting to process data for {data.Customer.Id}");
if (data.Customer.ProcessException != null)
{
log.Warning($"Unable to process {data.Customer.Id} please check the customer's last exception for more information.");
return;
}
if (scores?.Count > 0)
{
log.Info($"Importing {scores.Count} Secure Score entries for {data.Customer.Id}");
await secureScoreRepository.AddOrUpdateAsync(scores).ConfigureAwait(false);
}
if (alerts?.Count > 0)
{
log.Info($"Importing {alerts.Count} security alert entries for {data.Customer.Id}");
await securityAlertRepository.AddOrUpdateAsync(alerts).ConfigureAwait(false);
}
log.Info($"Successfully process data for {data.Customer.Id}");
}
public async Task <PipelineMessage> ProcessAsync(PipelineMessage message, TraceWriter traceWriter)
{
traceWriter.Warning("Appending strings to themselves");
var lines = message.Body.Split(new string[] { Environment.NewLine }, StringSplitOptions.None);
var sb = new StringBuilder();
foreach (var line in lines)
{
sb.AppendLine(line + "|" + line);
}
message.Body = sb.ToString();
return(message);
}
public static async Task <List <PartitionV2> > GetAllPartitions(TraceWriter log, CloudTableClient client, string owner)
{
try {
var table = client.GetTableReference(PartitionV2.TableName);
await table.CreateIfNotExistsAsync();
var query = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, owner);
var exQuery = new TableQuery <PartitionV2>().Where(query);
var partitions = await QueryHelper.ListAll(table, exQuery);
return(partitions);
}
catch (Exception ex) {
log.Warning(ex.Message, ex.ToString());
throw;
}
}
public static HttpResponseMessage GetEnrollment(
[HttpTrigger(AuthorizationLevel.Anonymous, "get",
Route = "Enrollment")] HttpRequestMessage req, TraceWriter log)
{
string email = UserManager.GetAuthenticatedEmail();
if (email == null)
{
log.Warning("Services.GetEnrollment unauthenciated user.");
}
log.Info(email);
StorageService service = new StorageService();
string list = service.GetEnrollment(email);
return(req.CreateResponse(HttpStatusCode.OK, list, "application/json"));
}
private static object GetWall(string wallId)
{
var meta = TableStorage.GetEntity <Walls>(Constants.TableNames.Walls, wallId, wallId);
string Title = "Untitled Wall";
if (meta == null)
{
_log.Warning($"The metadata for {wallId} in the Wall table was not found and should exist.");
}
else
{
Title = meta.Title;
}
return(new { BlobStorage.GetWall(wallId).Messages, Title });
}
private static string GetImageUri(string path, string html, TraceWriter log)
{
var parser = new HtmlParser();
var document = parser.ParseDocument(html);
var metaImage = document.QuerySelector("head > meta[property='og:image']") ??
document.QuerySelector("head > meta[property='twitter:image']");
string imageUrl;
if (!string.IsNullOrWhiteSpace(imageUrl = metaImage?.GetAttribute("content")))
{
return(imageUrl);
}
log.Warning($"No image meta tag found on '{path}'");
return(null);
}
private static async Task <AnalysisResult> AnalyseImage(string imageUrl, string VisionServiceApiKey, TraceWriter log)
{
VisionServiceClient visionServiceClient = new VisionServiceClient(VisionServiceApiKey);
List <VisualFeature> allVisualFeatures = new List <VisualFeature>
{
VisualFeature.Adult,
VisualFeature.Categories,
VisualFeature.Color,
VisualFeature.Description,
VisualFeature.Faces,
VisualFeature.ImageType,
VisualFeature.Tags
};
int retriesLeft = int.Parse(CloudConfigurationManager.GetSetting("CognitiveServicesRetryCount"));
int delay = int.Parse(CloudConfigurationManager.GetSetting("CognitiveServicesInitialRetryDelayms"));
AnalysisResult response = null;
while (true)
{
try
{
response = await visionServiceClient.AnalyzeImageAsync(imageUrl, allVisualFeatures);
break;
}
catch (ClientException exception) when(exception.HttpStatus == (HttpStatusCode)429 && retriesLeft > 0)
{
log.Info($"Computer Vision analysis call has been throttled or errored. {retriesLeft} retries left.");
if (retriesLeft == 1)
{
log.Warning($"Computer Vision analysis call still throttled or errored after {CloudConfigurationManager.GetSetting("CognitiveServicesRetryCount")} attempts, giving up.");
}
await Task.Delay(delay);
retriesLeft--;
delay *= 2;
continue;
}
}
return(response);
}
public async Task <PipelineMessage> ProcessAsync(PipelineMessage message, TraceWriter traceWriter)
{
traceWriter.Warning("Calculating average stats by class");
var typeDefinition = new[]
{
new {
Name = "",
Class = "",
Race = "",
Age = 0,
Level = 0,
Experience = 0,
Intelligence = 0,
Charisma = 0,
Wisdom = 0,
Dexterity = 0,
Strength = 0
}
};
var characters = JsonConvert.DeserializeAnonymousType(message.Body, typeDefinition);
var averages = characters
.GroupBy(x => x.Class)
.Select(x =>
(
Class: x.Key,
Count: x.Count(),
AverageIntelligence: x.Average(y => y.Intelligence),
AverageWisdom: x.Average(y => y.Wisdom),
AverageDexterity: x.Average(y => y.Dexterity),
AverageCharisma: x.Average(y => y.Charisma),
Averagestrength: x.Average(y => y.Strength)
));
foreach (var(Class, Count, AverageIntelligence, AverageWisdom, AverageDexterity, AverageCharisma, Averagestrength) in averages)
{
traceWriter.Warning($"- Class '{Class}' ({Count})");
traceWriter.Warning($"-- Int: {Math.Round(AverageIntelligence, 2)}");
traceWriter.Warning($"-- Dex: {Math.Round(AverageDexterity, 2)}");
traceWriter.Warning($"-- Cha: {Math.Round(AverageCharisma, 2)}");
traceWriter.Warning($"-- Wis: {Math.Round(AverageWisdom, 2)}");
traceWriter.Warning($"-- Str: {Math.Round(Averagestrength, 2)}");
}
return(message);
}
private static async Task <VersionedState <ZeroDataForPeriodAlertState> > UpdateState(TraceWriter log, string deviceId, decimal value, DateTime timestamp, string stateKey)
{
var versionedState = await Repository.LoadAsync <ZeroDataForPeriodAlertState>(stateKey);
bool succes = false;
bool dirty = false;
do
{
try
{
if (versionedState.State == null)
{
versionedState.State = new ZeroDataForPeriodAlertState()
{
DeviceId = deviceId
};
dirty = true;
}
if (value == 0 && !versionedState.State.ZeroSince.HasValue)
{
versionedState.State.ZeroSince = timestamp;
dirty = true;
}
else if (value > 0 && versionedState.State.ZeroSince.HasValue)
{
versionedState.State.ZeroSince = null;
dirty = true;
}
if (dirty)
{
await Repository.SaveAsync(stateKey, versionedState);
}
succes = true;
}
catch (StorageException ex)
{
log.Warning(ex.Message);
succes = false;
}
}while (!succes);
return(versionedState);
}
public static void Inputs(
[QueueTrigger("fakequeue1")] string triggerData,
[MobileTable(TableName = TableName, Id = "item1")] JObject item1,
[MobileTable(TableName = TableName, Id = "{QueueTrigger}")] JObject item2,
[MobileTable(Id = "item3")] TodoItem item3,
[MobileTable(Id = "{QueueTrigger}")] TodoItem item4,
TraceWriter trace)
{
Assert.NotNull(item1);
Assert.NotNull(item2);
Assert.NotNull(item3);
Assert.NotNull(item4);
// only modify item2
item2["Text"] = "changed";
trace.Warning("Inputs");
}
public static async Task <object> Run([HttpTrigger(WebHookType = "genericJson")] HttpRequestMessage req, TraceWriter log)
{
string jsonContent = await req.Content.ReadAsStringAsync();
log.Info(jsonContent);
try
{
dynamic request = JsonConvert.DeserializeObject <dynamic>(jsonContent)[0];
using (var original = new MemoryStream())
using (var thumb = new MemoryStream())
{
string url = request.data.url;
var filename = url.Substring(url.LastIndexOf('/') + 1);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("UseDevelopmentStorage=true");
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer inputContainer = blobClient.GetContainerReference("INPUT_CONTAINERNAME");
CloudBlockBlob originalBlob = inputContainer.GetBlockBlobReference(filename);
CloudBlobContainer outputContainer = blobClient.GetContainerReference("OUTPUT_CONTAINERNAME");
CloudBlockBlob blockBlob = outputContainer.GetBlockBlobReference($"thumb-{filename}");
await originalBlob.DownloadToStreamAsync(original);
original.Seek(0, SeekOrigin.Begin);
ImageBuilder.Current.Build(new ImageJob(original, thumb, _instructions));
thumb.Seek(0, SeekOrigin.Begin);
await blockBlob.UploadFromStreamAsync(thumb);
}
log.Info("Aaaaaaand we're done.");
return(req.CreateResponse(HttpStatusCode.OK, new
{
result = $"We're all done processing {request.data.url}!"
}));
}
catch (Exception e)
{
log.Warning(e.Message);
return(req.CreateResponse(HttpStatusCode.BadRequest, e));
}
}
public static void Run([QueueTrigger("pokemonValidatorQueue")] Pokemon pokemon, TraceWriter log)
{
log.Info($"Validating Pokemon data");
var validatorService = new ValidatorService();
var isValidPokemon = validatorService.ValidatePokemon(pokemon);
if (isValidPokemon)
{
log.Info("Valid Pokemon");
}
else
{
log.Warning("Invalid Pokemon");
}
log.Info($"Data validator function processed");
}
private static string ExtractTextFromEmbeddedDocuments(
WordprocessingDocument element,
TraceWriter log)
{
var builder = new StringBuilder();
try
{
foreach (var part in element.Parts)
{
var testForEmbedding = part.OpenXmlPart.GetPartsOfType <EmbeddedPackagePart>();
foreach (EmbeddedPackagePart embedding in testForEmbedding)
{
string fileName = embedding.Uri.OriginalString.Split('/').Last();
using (var stream = embedding.GetStream())
{
switch (Path.GetExtension(fileName))
{
case ".xlsx":
builder.Append(GetAllTextFromExcelDoc(stream, log));
break;
case ".docx":
builder.Append(GetAllTextFromWordDoc(stream, log));
break;
default:
break;
}
}
}
}
}
catch (ApplicationException aex)
{
log.Warning($"Word doc contained embedded documents, but text could not be extracted. Error:{aex.Message}");
}
return(builder.ToString());
}
static async Task ProcessResourceGroups(IEnumerable <string> requiredTagsList, List <InvalidTagResource> invalidTypes, string subscriptionId, AuditStats stats)
{
var resourceGroups = await _resourceManager.GetResourceGroups(subscriptionId);
stats.ResourceGroupsTotal = resourceGroups.Count;
foreach (var rg in resourceGroups)
{
_log.Info("*** Resource Group: " + rg.Name);
var tagsToSync = TagService.GetRequiredTags((Dictionary <string, string>)rg.Tags, requiredTagsList);
if (tagsToSync.Count < 1)
{
_log.Warning("Resource group: " + rg.Name + " does not have required tags.");
stats.ResourceGroupsSkipped += 1;
}
else
{
List <ResourceItem> resources = await _resourceManager.GetResources(rg.Name, subscriptionId, invalidTypes.Select(t => t.Type).ToList());
stats.ResourceItemsTotal = resources.Count();
foreach (var resource in resources)
{
var result = TagService.GetTagUpdates(resource.Tags.ToDictionary(x => x.Key, x => x.Value), tagsToSync);
if (result.Count > 0)
{
stats.ResourceItemsWithUpdates += 1;
resource.Tags = result;
string messageText = JsonConvert.SerializeObject(resource);
_log.Info("Requesting tags for: " + resource.Id);
_outQueue.Add(messageText);
}
else
{
stats.ResourceItemsSkipped += 1;
}
}
}
}
}
public static async Task <HttpResponseMessage> Approve(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = "Approval/{id}")]
HttpRequestMessage req,
[OrchestrationClient] DurableOrchestrationClient client,
[Table("Approvals", "Approval", "{id}", Connection = "azureWebJobsStorage")] Approval approval,
TraceWriter log)
{
string result = req.RequestUri.ParseQueryString()["result"];
if (result == null)
{
return(req.CreateResponse(HttpStatusCode.BadRequest, "Need an approval result"));
}
log.Warning($"Sending approval result to {approval.OrchestrationId} of {result}");
await client.RaiseEventAsync(approval.OrchestrationId, "EVENT_APPROVAL", result);
return(req.CreateResponse(HttpStatusCode.OK));
}
public static async Task <Tuple <bool, string> > PostMessageAsync(string url, string messageAsJson, TraceWriter log)
{
var content = new System.Net.Http.StringContent(messageAsJson, System.Text.Encoding.UTF8, "application/json");
// Prepare the retry policy
var response = await Policy
.HandleResult <HttpResponseMessage>(message => !message.IsSuccessStatusCode)
.WaitAndRetryAsync(3, i => TimeSpan.FromSeconds(2), (result, timeSpan, retryCount, context) =>
{
log.Warning($"Request failed with {result.Result.StatusCode}. Waiting {timeSpan} before next retry. Retry attempt {retryCount}");
Console.WriteLine("Retrying...");
})
.ExecuteAsync(() => httpClient.PostAsync(new Uri(url), content));
bool sucessStatusCode = response.IsSuccessStatusCode;
string wfResponse = await response.Content.ReadAsStringAsync();
Tuple <bool, string> returnValue = new Tuple <bool, string>(sucessStatusCode, wfResponse);
return(returnValue);
}
public static async Task <HttpResponseMessage> SubmitVideoApproval(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "SubmitVideoApproval/{id}")]
HttpRequestMessage req, [OrchestrationClient] DurableOrchestrationClient client,
[Table("Approvals", "Approval", "{id}", Connection = "AzureWebJobsStorage")] Approval approval,
TraceWriter log)
{
// nb if the approval code doesn't exist, framework just returns a 404 before we get here
string result = req.GetQueryNameValuePairs()
.FirstOrDefault(q => string.Compare(q.Key, "result", true) == 0).Value;
if (result == null)
{
return(req.CreateResponse(HttpStatusCode.BadRequest, "Need an approval result"));
}
log.Warning($"Sending approval result to {approval.OrchestrationId} of {result}");
// send the ApprovalResult external event to this orchestration
await client.RaiseEventAsync(approval.OrchestrationId, "ApprovalResult", result);
return(req.CreateResponse(HttpStatusCode.OK));
}
public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "tv/{id}/commands/-/get")]
HttpRequestMessage req,
string id,
TraceWriter log
)
{
var id2 = new Guid(id);
var queue = Storage.GetCommandQueueReference(id2);
_GetMessage:
var msg = await queue.GetMessageAsync(TimeSpan.FromMinutes(2), null, null);
if (msg == null)
{
return(req.CreateResponse <Result>(ErrorCode.TvCommandQueueIsEmpty));
}
if (!(msg.DequeueCount <= 5))
{
// dead letter
log.Warning($"message '{msg.Id}' dead letter");
await queue.DeleteMessageAsync(msg);
goto _GetMessage;
}
var command = JObject.Parse(msg.AsString);
return(req.CreateResponse(new Result <CommandResponse>
{
Payload = new CommandResponse
{
Command = command,
CompleteUrl = new Uri(req.RequestUri, $"/api/tv/{id}/commands/{Uri.EscapeDataString(msg.Id)}/delete?popReceipt={Uri.EscapeDataString(msg.PopReceipt)}")
}
}));
}
public static string Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequestMessage req, TraceWriter log)
{
NarsHttpResponseObject result = new NarsHttpResponseObject();
log.Info("GetNarsUser HttpTrigger executing...");
var content = req.Content.ReadAsStringAsync().Result;
var response = NarsCaller.Execute(Environment.GetEnvironmentVariable("userUrl"), content);
var responseResult = response.Content.ReadAsStringAsync().Result;
if (response.IsSuccessStatusCode)
{
log.Info("****SUCCESS****");
JObject respResult = (JObject)JsonConvert.DeserializeObject(responseResult);
result = new NarsHttpResponseObject
{
ReturnType = "NarsUser",
ReturnObject = JsonConvert.DeserializeObject<NarsUser>(respResult["d"].ToString()),
Success = true,
Exception = null
};
}
else
{
log.Warning(responseResult);
result = new NarsHttpResponseObject
{
ReturnType = "HttpResponse.Content.ToString",
ReturnObject = responseResult,
Success = false,
Exception = null
};
}
return JsonConvert.SerializeObject(result);
}
protected override bool RejectAllRequests()
{
if (base.RejectAllRequests())
{
return(true);
}
if (Config.DynamicThrottlesEnabled &&
((DateTime.UtcNow - _lastPerformanceCheck) > TimeSpan.FromSeconds(_performanceCheckPeriodSeconds)))
{
// only check host status periodically
Collection <string> exceededCounters = new Collection <string>();
_rejectAllRequests = _performanceManager.IsUnderHighLoad(exceededCounters);
_lastPerformanceCheck = DateTime.UtcNow;
if (_rejectAllRequests)
{
TraceWriter.Warning($"Thresholds for the following counters have been exceeded: [{string.Join(", ", exceededCounters)}]");
}
}
return(_rejectAllRequests);
}
public void Emit(LogEvent logEvent)
{
switch (logEvent.Level)
{
case LogEventLevel.Verbose:
case LogEventLevel.Debug:
writer.Verbose(logEvent.RenderMessage());
break;
case LogEventLevel.Information:
writer.Info(logEvent.RenderMessage());
break;
case LogEventLevel.Warning:
writer.Warning(logEvent.RenderMessage());
break;
default:
writer.Error(logEvent.RenderMessage());
break;
}
}
public static async Task Run(
[TimerTrigger("0 0 3 * * *")] TimerInfo myTimer,
TraceWriter log,
[StorageAccount("StorageConnectionString")] CloudStorageAccount storageAccount
)
{
log.Info($"Performing cleanup - {DateTime.Now}");
var client = storageAccount.CreateCloudTableClient();
new ThrottleService(client, log).Clear();
CloudTable table = client.GetTableReference(EmailAuthToken.TableName);
TableQuery <EmailAuthToken> rangeQuery = new TableQuery <EmailAuthToken>().Where(
TableQuery.GenerateFilterConditionForDate("Timestamp",
QueryComparisons.LessThan,
DateTime.UtcNow.AddHours(-5)
)
);
try {
// TODO: Group by PartitionKey and delete in batches.
await table.CreateIfNotExistsAsync();
var entities = await QueryHelper.ListAll(table, rangeQuery);
log.Info($"Got {entities.Count} entities for deletion");
foreach (var entity in entities)
{
await table.ExecuteAsync(TableOperation.Delete(entity));
}
}
catch (Exception ex) {
log.Warning(ex.Message, ex.Source);
throw;
}
log.Info("Cleanup completed");
}
public static async Task <HttpResponseMessage> SubmitVideoApproval(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "SubmitVideoApproval/{id}")]
HttpRequestMessage req,
[OrchestrationClient] DurableOrchestrationClient client,
[Table("Approvals", "Approval", "{id}", Connection = "AzureWebJobsStorage")]
Approval approval,
TraceWriter log)
{
var result = req.GetQueryNameValuePairs().FirstOrDefault(x => x.Key == "result").Value;
if (result == null)
{
return(req.CreateResponse(HttpStatusCode.BadRequest, "Need an approval result"));
}
log.Warning($"Sending approval result to {approval.OrchestrationId} of {result}");
// send the ApprovalResult external event to this orchestration
await client.RaiseEventAsync(approval.OrchestrationId, Constants.ApprovalResultEventName, result);
return(req.CreateResponse(HttpStatusCode.OK));
}
private static async Task ProcessLine(string name, string line, int lineNumber, TraceWriter log)
{
if (string.IsNullOrWhiteSpace(line))
{
log.Warning($"{name}: {lineNumber} is empty.");
return;
}
var parts = line.Split(',');
if (parts.Length != 8)
{
log.Error($"{name}: {lineNumber} invalid data: {line}.");
return;
}
var item = new StockSecurity()
{
Symbol = parts[0]
};
//if ((int.TryParse(parts[1], out int complete) == false) || complete < 0 || complete > 1)
//{
// log.Error($"{name}: {lineNumber} bad complete flag: {parts[1]}.");
//}
using (var context = new SignalContext())
{
if (context.StockSecurities.Any(todo => todo.Symbol == item.Symbol))
{
log.Error($"{name}: {lineNumber} duplicate Stock Security: \"{item.Symbol}\".");
return;
}
context.StockSecurities.Add(item);
await context.SaveChangesAsync();
log.Info($"{name}: {lineNumber} inserted Stock Security: \"{item.Symbol}\" with id: {item.Id}.");
}
}
public static async Task <string> RegisterNewDevice(dynamic queueItem, TraceWriter log)
{
using (var client = new HttpClient())
{
string resultContent = null;
try
{
string url = SetupApiCall(client, log) + "/devices";
dynamic item = new ExpandoObject();
item.deveui = queueItem.deviceId;
StringContent httpConent = new StringContent(JsonConvert.SerializeObject(item), Encoding.UTF8, "application/json");
var result = await client.PostAsync(url, httpConent);
resultContent = await result.Content.ReadAsStringAsync();
if (!result.IsSuccessStatusCode)
{
//TODO: at the moment Loriot doesn't send htis status if the sensor just exists
if (result.StatusCode == HttpStatusCode.Conflict)
{
log.Warning(String.Format("Sensor just exists in Loriot"));
}
else
{
//HACK: check if is duplicate record error
if (!resultContent.Contains("E11000 duplicate key error"))
{
throw new HttpRequestException(result.ReasonPhrase);
}
}
}
}
catch (ConfigurationException confEx)
{
}
return(resultContent);
}
}
