//[Function("QueueNodeHearbeat")]
public async Async.Task Run([QueueTrigger("node-heartbeat", Connection = "AzureWebJobsStorage")] string msg)
{
_log.Info($"heartbeat: {msg}");
var nodes = _context.NodeOperations;
var events = _context.Events;
var hb = JsonSerializer.Deserialize <NodeHeartbeatEntry>(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}");
var node = await nodes.GetByMachineId(hb.NodeId);
var log = _log.WithTag("NodeId", hb.NodeId.ToString());
if (node == null)
{
log.Warning($"invalid node id: {hb.NodeId}");
return;
}
var newNode = node with {
Heartbeat = DateTimeOffset.UtcNow
};
var r = await nodes.Replace(newNode);
if (!r.IsOk)
{
var(status, reason) = r.ErrorV;
log.Error($"Failed to replace heartbeat info due to [{status}] {reason}");
}
// TODO: do we still send event if we fail do update the table ?
await events.SendEvent(new EventNodeHeartbeat(node.MachineId, node.ScalesetId, node.PoolName));
}
}
// Not converting to bytes, as it's not neccessary in C#. Just keeping as string.
public async Async.Task <Tuple <string, string?> > BuildMessage(Guid webhookId, Guid eventId, EventType eventType, BaseEvent webhookEvent, String?secretToken, WebhookMessageFormat?messageFormat)
{
string data = "";
if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid)
{
var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookEvent, DataVersion: "1.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) };
data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions());
}
else
{
var instanceId = await _context.Containers.GetInstanceId();
var webhookMessage = new WebhookMessage(WebhookId: webhookId, EventId: eventId, EventType: eventType, Event: webhookEvent, InstanceId: instanceId, InstanceName: _context.Creds.GetInstanceName());
data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions());
}
string?digest = null;
var hmac = HMAC.Create("HMACSHA512");
if (secretToken != null && hmac != null)
{
hmac.Key = System.Text.Encoding.UTF8.GetBytes(secretToken);
digest = Convert.ToHexString(hmac.ComputeHash(System.Text.Encoding.UTF8.GetBytes(data)));
}
return(new Tuple <string, string?>(data, digest));
}
public async Task <HttpResponseData> Patch([HttpTrigger(AuthorizationLevel.Anonymous, "patch", Route = "testhooks/instance-config")] HttpRequestData req)
{
_log.Info("Patch instance config");
var s = await req.ReadAsStringAsync();
var newInstanceConfig = JsonSerializer.Deserialize <InstanceConfig>(s !, EntityConverter.GetJsonSerializerOptions());
if (newInstanceConfig is null)
{
var resp = req.CreateResponse();
resp.StatusCode = HttpStatusCode.BadRequest;
await resp.WriteAsJsonAsync(new { Error = "Instance config is not set" });
return(resp);
}
else
{
var query = UriExtension.GetQueryComponents(req.Url);
bool isNew = UriExtension.GetBool("isNew", query, false);
//requireEtag wont' work since our current schema does not return etag to the client when getting data form the table, so
// there is no way to know which etag to use
bool requireEtag = UriExtension.GetBool("requireEtag", query, false);
await _configOps.Save(newInstanceConfig, isNew, requireEtag);
var resp = req.CreateResponse();
resp.StatusCode = HttpStatusCode.OK;
return(resp);
}
}
public async Task <HttpResponseData> CreateNode([HttpTrigger(AuthorizationLevel.Anonymous, "put", Route = "testhooks/nodeOperations/node")] HttpRequestData req)
{
_log.Info("create node");
var query = UriExtension.GetQueryComponents(req.Url);
Guid poolId = Guid.Parse(query["poolId"]);
var poolName = PoolName.Parse(query["poolName"]);
Guid machineId = Guid.Parse(query["machineId"]);
Guid?scaleSetId = default;
if (query.ContainsKey("scaleSetId"))
{
scaleSetId = Guid.Parse(query["scaleSetId"]);
}
string version = query["version"];
bool isNew = UriExtension.GetBool("isNew", query, false);
var node = await _nodeOps.Create(poolId, poolName, machineId, scaleSetId, version, isNew);
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteAsJsonAsync(JsonSerializer.Serialize(node, EntityConverter.GetJsonSerializerOptions()));
return(resp);
}
public async Task <HttpResponseData> SearchStates([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/searchStates")] HttpRequestData req)
{
_log.Info("search states");
var query = UriExtension.GetQueryComponents(req.Url);
Guid?poolId = UriExtension.GetGuid("poolId", query);
Guid?scaleSetId = UriExtension.GetGuid("scaleSetId", query);
List <NodeState>?states = default;
if (query.ContainsKey("states"))
{
states = query["states"].Split('-').Select(s => Enum.Parse <NodeState>(s)).ToList();
}
string?poolNameString = UriExtension.GetString("poolName", query);
PoolName?poolName = poolNameString is null ? null : PoolName.Parse(poolNameString);
var excludeUpdateScheduled = UriExtension.GetBool("excludeUpdateScheduled", query, false);
int?numResults = UriExtension.GetInt("numResults", query);
var r = _nodeOps.SearchStates(poolId, scaleSetId, states, poolName, excludeUpdateScheduled, numResults);
var json = JsonSerializer.Serialize(await r.ToListAsync(), EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(json);
return(resp);
}
public async Task <IList <T> > PeekQueue <T>(string name, StorageType storageType)
{
var client = await GetQueueClient(name, storageType);
var result = new List <T>();
var msgs = await client.PeekMessagesAsync(client.MaxPeekableMessages);
if (msgs is null)
{
return(result);
}
else if (msgs.GetRawResponse().IsError)
{
_log.Error($"failed to peek messages due to {msgs.GetRawResponse().ReasonPhrase}");
return(result);
}
else
{
foreach (var msg in msgs.Value)
{
var obj = JsonSerializer.Deserialize <T>(msg.Body.ToString(), EntityConverter.GetJsonSerializerOptions());
if (obj is not null)
{
result.Add(obj);
}
}
}
return(result);
}
//[Function("QueueFileChanges")]
public async Async.Task Run(
[QueueTrigger("file-changes-refactored", Connection = "AzureWebJobsStorage")] string msg,
int dequeueCount)
{
var fileChangeEvent = JsonSerializer.Deserialize <JsonDocument>(msg, EntityConverter.GetJsonSerializerOptions());
var lastTry = dequeueCount == MAX_DEQUEUE_COUNT;
var _ = fileChangeEvent ?? throw new ArgumentException("Unable to parse queue trigger as JSON");
// check type first before calling Azure APIs
const string eventType = "eventType";
if (!fileChangeEvent.RootElement.TryGetProperty(eventType, out var eventTypeElement) ||
eventTypeElement.GetString() != "Microsoft.Storage.BlobCreated")
{
return;
}
const string topic = "topic";
if (!fileChangeEvent.RootElement.TryGetProperty(topic, out var topicElement) ||
!_storage.CorpusAccounts().Contains(topicElement.GetString()))
{
return;
}
await file_added(_log, fileChangeEvent, lastTry);
}
public async Task <SecretData <SecretAddress>?> SaveToKeyvault <T>(SecretData <T> secretData)
{
if (secretData == null || secretData.Secret is null)
{
return(null);
}
if (secretData.Secret is SecretAddress)
{
return(secretData as SecretData <SecretAddress>);
}
else
{
var secretName = Guid.NewGuid();
string secretValue;
if (secretData.Secret is string)
{
secretValue = (secretData.Secret as string) !.Trim();
}
else
{
secretValue = JsonSerializer.Serialize(secretData.Secret, EntityConverter.GetJsonSerializerOptions());
}
var kv = await StoreInKeyvault(GetKeyvaultAddress(), secretName.ToString(), secretValue);
return(new SecretData <SecretAddress>(new SecretAddress(kv.Id)));
}
}
public async Async.Task Run([QueueTrigger("proxy", Connection = "AzureWebJobsStorage")] string msg)
{
_log.Info($"heartbeat: {msg}");
var hb = JsonSerializer.Deserialize <ProxyHeartbeat>(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}");;
var newHb = hb with {
TimeStamp = DateTimeOffset.UtcNow
};
var proxy = await _proxy.GetByProxyId(newHb.ProxyId);
var log = _log.WithTag("ProxyId", newHb.ProxyId.ToString());
if (proxy == null)
{
log.Warning($"invalid proxy id: {newHb.ProxyId}");
return;
}
var newProxy = proxy with {
Heartbeat = newHb
};
var r = await _proxy.Replace(newProxy);
if (!r.IsOk)
{
var(status, reason) = r.ErrorV;
log.Error($"Failed to replace proxy heartbeat record due to [{status}] {reason}");
}
}
}
public async Task <HttpResponseData> GetPool([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/poolOperations/pool")] HttpRequestData req)
{
_log.Info("get pool");
var query = UriExtension.GetQueryComponents(req.Url);
var poolRes = await _poolOps.GetByName(PoolName.Parse(query["name"]));
if (poolRes.IsOk)
{
var resp = req.CreateResponse(HttpStatusCode.OK);
var data = poolRes.OkV;
var msg = JsonSerializer.Serialize(data, EntityConverter.GetJsonSerializerOptions());
await resp.WriteStringAsync(msg);
return(resp);
}
else
{
var resp = req.CreateResponse(HttpStatusCode.BadRequest);
var msg = JsonSerializer.Serialize(poolRes.ErrorV, EntityConverter.GetJsonSerializerOptions());
await resp.WriteStringAsync(msg);
return(resp);
}
}
//[Function("QueueWebhooks")]
public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg)
{
_log.Info($"Webhook Message Queued: {msg}");
var obj = JsonSerializer.Deserialize <WebhookMessageQueueObj>(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}");
await _webhookMessageLog.ProcessFromQueue(obj);
}
public void LogEvent(BaseEvent anEvent)
{
var options = EntityConverter.GetJsonSerializerOptions();
options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
options.Converters.Add(new RemoveUserInfo());
var serializedEvent = JsonSerializer.Serialize(anEvent, anEvent.GetType(), options);
_log.WithTag("Event Type", anEvent.GetEventType().ToString()).Info($"sending event: {anEvent.GetEventType()} - {serializedEvent}");
}
public async Task <HttpResponseData> GetQueueTasksTestHook([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/notificationOperations/getQueueTasks")] HttpRequestData req)
{
_log.Info("get queue tasks");
var queueuTasks = _notificationOps.GetQueueTasks();
var json = JsonSerializer.Serialize(await queueuTasks.ToListAsync(), EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(json);
return(resp);
}
public async Task <HttpResponseData> SearchExpired([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/jobOps/searchExpired")] HttpRequestData req)
{
_log.Info("Search expired jobs");
var jobs = await _jobOps.SearchExpired().ToListAsync();
var msg = JsonSerializer.Serialize(jobs, EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(msg);
return(resp);
}
public async Task <HttpResponseData> GetMonitorSettings([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/logAnalytics/monitorSettings")] HttpRequestData req)
{
_log.Info("Get monitor settings");
var monitorSettings = await _logAnalytics.GetMonitorSettings();
var msg = JsonSerializer.Serialize(monitorSettings, EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(msg);
return(resp);
}
public async Task <HttpResponseData> DeleteNode([HttpTrigger(AuthorizationLevel.Anonymous, "delete", Route = "testhooks/nodeOperations/node")] HttpRequestData req)
{
_log.Info("delete node");
var s = await req.ReadAsStringAsync();
var node = JsonSerializer.Deserialize <Node>(s !, EntityConverter.GetJsonSerializerOptions());
var r = _nodeOps.Delete(node !);
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteAsJsonAsync(r);
return(resp);
}
public async Task <T?> GetSecretObj <T>(Uri secretUrl)
{
var secret = await GetSecret(secretUrl);
if (secret is null)
{
return(default(T));
}
else
{
return(JsonSerializer.Deserialize <T>(secret.Value, EntityConverter.GetJsonSerializerOptions()));
}
}
public async Task <HttpResponseData> ListInstanceIds([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/vmssOperations/listInstanceIds")] HttpRequestData req)
{
_log.Info("list instance ids");
var query = UriExtension.GetQueryComponents(req.Url);
var name = UriExtension.GetGuid("name", query) ?? throw new Exception("name must be set");
var ids = await _vmssOps.ListInstanceIds(name);
var json = JsonSerializer.Serialize(ids, EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(json);
return(resp);
}
public async Task <HttpResponseData> SendEvent([HttpTrigger(AuthorizationLevel.Anonymous, "put", Route = "testhooks/events/sendEvent")] HttpRequestData req)
{
_log.Info("Send event");
var s = await req.ReadAsStringAsync();
var msg = JsonSerializer.Deserialize <EventMessage>(s !, EntityConverter.GetJsonSerializerOptions());
await _events.SendEvent(msg !.Event);
var resp = req.CreateResponse(HttpStatusCode.OK);
return(resp);
}
public Events(IQueue queue, IWebhookOperations webhook, ILogTracer log, IContainers containers, ICreds creds)
{
_queue = queue;
_webhook = webhook;
_log = log;
_containers = containers;
_creds = creds;
_options = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions())
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
_options.Converters.Add(new RemoveUserInfo());
}
public async Task <HttpResponseData> MarkTasksStoppedEarly([HttpTrigger(AuthorizationLevel.Anonymous, "patch", Route = "testhooks/nodeOperations/markTasksStoppedEarly")] HttpRequestData req)
{
_log.Info("mark tasks stopped early");
var s = await req.ReadAsStringAsync();
var markTasks = JsonSerializer.Deserialize <MarkTasks>(s !, EntityConverter.GetJsonSerializerOptions());
await _nodeOps.MarkTasksStoppedEarly(markTasks !.node, markTasks.error);
var resp = req.CreateResponse(HttpStatusCode.OK);
return(resp);
}
public async Task <HttpResponseData> GetRegressionReportTask([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/notificationOperations/getRegressionReportTask")] HttpRequestData req)
{
_log.Info("get regression report task");
var s = await req.ReadAsStringAsync();
var report = JsonSerializer.Deserialize <RegressionReport>(s !, EntityConverter.GetJsonSerializerOptions());
var task = (_notificationOps as NotificationOperations) !.GetRegressionReportTask(report !);
var json = JsonSerializer.Serialize(task, EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(json);
return(resp);
}
public async Task <HttpResponseData> UpdateScaleInProtection([HttpTrigger(AuthorizationLevel.Anonymous, "put", Route = "testhooks/vmssOperations/updateScaleInProtection")] HttpRequestData req)
{
_log.Info("list instance ids");
var query = UriExtension.GetQueryComponents(req.Url);
var name = UriExtension.GetGuid("name", query) ?? throw new Exception("name must be set");
var vmId = UriExtension.GetGuid("vmId", query) ?? throw new Exception("vmId must be set");
var protectFromScaleIn = UriExtension.GetBool("protectFromScaleIn", query);
var id = await _vmssOps.UpdateScaleInProtection(name, vmId, protectFromScaleIn);
var json = JsonSerializer.Serialize(id, EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(json);
return(resp);
}
public async Task <HttpResponseData> GetJob([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/jobOps/job")] HttpRequestData req)
{
_log.Info("Get job info");
var query = UriExtension.GetQueryComponents(req.Url);
var jobId = Guid.Parse(query["jobId"]);
var job = await _jobOps.Get(jobId);
var msg = JsonSerializer.Serialize(job, EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(msg);
return(resp);
}
public async Task <HttpResponseData> GetByMachineId([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/nodeOperations/getByMachineId")] HttpRequestData req)
{
_log.Info("Get node by machine id");
var query = UriExtension.GetQueryComponents(req.Url);
var machineId = query["machineId"];
var node = await _nodeOps.GetByMachineId(Guid.Parse(machineId));
var msg = JsonSerializer.Serialize(node, EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(msg);
return(resp);
}
public async Task <HttpResponseData> CanProcessNewWork([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/canProcessNewWork")] HttpRequestData req)
{
_log.Info("Can process new work");
var s = await req.ReadAsStringAsync();
var node = JsonSerializer.Deserialize <Node>(s !, EntityConverter.GetJsonSerializerOptions());
var r = await _nodeOps.CanProcessNewWork(node !);
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteAsJsonAsync(r);
return(resp);
}
public async Task <HttpResponseData> GetNotifications([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/notificationOperations/getNotifications")] HttpRequestData req)
{
_log.Info("get notifications");
var s = await req.ReadAsStringAsync();
var query = UriExtension.GetQueryComponents(req.Url);
var container = query["container"];
var notifications = _notificationOps.GetNotifications(new Container(container));
var json = JsonSerializer.Serialize(await notifications.ToListAsync(), EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(json);
return(resp);
}
public async Task <HttpResponseData> GetDeadNodes([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/getDeadNodes")] HttpRequestData req)
{
_log.Info("get dead nodes");
var query = UriExtension.GetQueryComponents(req.Url);
Guid scaleSetId = Guid.Parse(query["scaleSetId"]);
TimeSpan timeSpan = TimeSpan.Parse(query["timeSpan"]);
var nodes = await(_nodeOps.GetDeadNodes(scaleSetId, timeSpan).ToListAsync());
var json = JsonSerializer.Serialize(nodes, EntityConverter.GetJsonSerializerOptions());
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(json);
return(resp);
}
private IReport?ParseReportOrRegression(string content, string?filePath, bool expectReports = false)
{
var regressionReport = JsonSerializer.Deserialize <RegressionReport>(content, EntityConverter.GetJsonSerializerOptions());
if (regressionReport == null || regressionReport.CrashTestResult == null)
{
var report = JsonSerializer.Deserialize <Report>(content, EntityConverter.GetJsonSerializerOptions());
if (expectReports && report == null)
{
_log.Error($"unable to parse report ({filePath}) as a report or regression");
return(null);
}
return(report);
}
return(regressionReport);
}
public async Task <HttpResponseData> ToReimage([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/toReimage")] HttpRequestData req)
{
_log.Info("to reimage");
var query = UriExtension.GetQueryComponents(req.Url);
var done = UriExtension.GetBool("done", query, false);
var s = await req.ReadAsStringAsync();
var node = JsonSerializer.Deserialize <Node>(s !, EntityConverter.GetJsonSerializerOptions());
var r = _nodeOps.ToReimage(node !, done);
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteAsJsonAsync(r);
return(resp);
}
