public async Async.Task <RegressionReportOrReport?> GetReportOrRegression(Container container, string fileName, bool expectReports = false, params string[] args)
{
var filePath = String.Join("/", new[] { container.ContainerName, fileName });
if (!fileName.EndsWith(".json"))
{
if (expectReports)
{
_log.Error($"get_report invalid extension: {filePath}");
}
return(null);
}
var blob = await _containers.GetBlob(container, fileName, StorageType.Corpus);
if (blob == null)
{
if (expectReports)
{
_log.Error($"get_report invalid blob: {filePath}");
}
return(null);
}
return(ParseReportOrRegression(blob.ToString(), filePath, expectReports));
}
async Async.Task SetState(Scaleset scaleSet, ScalesetState state)
{
if (scaleSet.State == state)
{
return;
}
if (scaleSet.State == ScalesetState.Halt)
{
return;
}
var updatedScaleSet = scaleSet with {
State = state
};
var r = await this.Replace(updatedScaleSet);
if (!r.IsOk)
{
_log.Error($"Failed to update scaleset {scaleSet.ScalesetId} when updating state from {scaleSet.State} to {state}");
}
if (state == ScalesetState.Resize)
{
await _context.Events.SendEvent(
new EventScalesetResizeScheduled(updatedScaleSet.ScalesetId, updatedScaleSet.PoolName, updatedScaleSet.Size)
);
}
else
{
await _context.Events.SendEvent(
new EventScalesetStateUpdated(updatedScaleSet.ScalesetId, updatedScaleSet.PoolName, updatedScaleSet.State)
);
}
}
async Async.Task SetFailed(Scaleset scaleSet, Error error)
{
if (scaleSet.Error is not null)
{
return;
}
await SetState(scaleSet with {
Error = error
}, ScalesetState.CreationFailed);
await _context.Events.SendEvent(new EventScalesetFailed(scaleSet.ScalesetId, scaleSet.PoolName, error));
}
public async Task <BlobContainerClient?> GetOrCreateContainerClient(Container container, StorageType storageType, IDictionary <string, string>?metadata)
{
var containerClient = await FindContainer(container, StorageType.Corpus);
if (containerClient is not null)
{
return(containerClient);
}
var account = _storage.ChooseAccount(storageType);
var client = await _storage.GetBlobServiceClientForAccount(account);
var containerName = _config.OneFuzzStoragePrefix + container.ContainerName;
var cc = client.GetBlobContainerClient(containerName);
try {
await cc.CreateAsync(metadata : metadata);
} catch (RequestFailedException ex) when(ex.ErrorCode == "ContainerAlreadyExists")
{
// note: resource exists error happens during creation if the container
// is being deleted
_log.Error($"unable to create container. account: {account} container: {container.ContainerName} metadata: {metadata} - {ex.Message}");
return(null);
}
return(cc);
}
public async Async.Task <IDictionary <Guid, string> > ListInstanceIds(Guid name)
{
_log.Verbose($"get instance IDs for scaleset {name}");
var results = new Dictionary <Guid, string>();
VirtualMachineScaleSetResource res;
try {
var r = await GetVmssResource(name).GetAsync();
res = r.Value;
} catch (Exception ex) when(ex is RequestFailedException)
{
_log.Verbose($"vm does not exist {name}");
return(results);
}
if (res is null)
{
_log.Verbose($"vm does not exist {name}");
return(results);
}
else
{
try {
await foreach (var instance in res !.GetVirtualMachineScaleSetVms().AsAsyncEnumerable())
{
if (instance is not null)
{
Guid key;
if (Guid.TryParse(instance.Data.VmId, out key))
{
results[key] = instance.Data.InstanceId;
}
else
{
_log.Error($"failed to convert vmId {instance.Data.VmId} to Guid");
}
}
}
} catch (Exception ex) when(ex is RequestFailedException || ex is CloudException)
{
_log.Exception(ex, $"vm does not exist {name}");
}
}
return(results);
}
public async Task <OneFuzzResult <bool> > AddExtensions(Vm vm, Dictionary <string, VirtualMachineExtensionData> extensions)
{
var status = new List <string>();
var toCreate = new List <KeyValuePair <string, VirtualMachineExtensionData> >();
foreach (var extensionConfig in extensions)
{
var extensionName = extensionConfig.Key;
var extensionData = extensionConfig.Value;
var extension = await GetExtension(vm.Name, extensionName);
if (extension != null)
{
_logTracer.Info(
$"vm extension state: {vm.Name} - {extensionName} - {extension.ProvisioningState}"
);
status.Add(extension.ProvisioningState);
}
else
{
toCreate.Add(extensionConfig);
}
}
if (toCreate.Any())
{
foreach (var config in toCreate)
{
await CreateExtension(vm.Name, config.Key, config.Value);
}
}
else
{
if (status.All(s => string.Equals(s, "Succeeded", StringComparison.Ordinal)))
{
return(OneFuzzResult <bool> .Ok(true));
}
else if (status.Any(s => string.Equals(s, "Failed", StringComparison.Ordinal)))
{
return(OneFuzzResult <bool> .Error(
ErrorCode.VM_CREATE_FAILED,
"failed to launch extension"
));
}
else if (!(status.Contains("Creating") || status.Contains("Updating")))
{
_logTracer.Error($"vm agent - unknown state {vm.Name}: {JsonConvert.SerializeObject(status)}");
}
}
return(OneFuzzResult <bool> .Ok(false));
}
public async Async.Task SyncScalesetSize(Scaleset scaleset)
{
// # If our understanding of size is out of sync with Azure, resize the
// # scaleset to match our understanding.
if (scaleset.State != ScalesetState.Running)
{
return;
}
var size = await _context.VmssOperations.GetVmssSize(scaleset.ScalesetId);
if (size is null)
{
_log.Info($"{SCALESET_LOG_PREFIX} scaleset is unavailable. scaleset_id: {scaleset.ScalesetId}");
//#if the scaleset is missing, this is an indication the scaleset
//# was manually deleted, rather than having OneFuzz delete it. As
//# such, we should go thruogh the process of deleting it.
await SetShutdown(scaleset, now : true);
return;
}
if (size != scaleset.Size)
{
//# Azure auto-scaled us or nodes were manually added/removed
//# New node state will be synced in cleanup_nodes
_log.Info($"{SCALESET_LOG_PREFIX} unexpected scaleset size, resizing. scaleset_id: {scaleset.ScalesetId} expected:{scaleset.Size} actual:{size}");
scaleset = scaleset with {
Size = size.Value
};
var replaceResult = await Update(scaleset);
if (!replaceResult.IsOk)
{
_log.Error($"Failed to update scaleset size for scaleset {scaleset.ScalesetId} due to {replaceResult.ErrorV}");
}
}
}
private async Async.Task <BlobServiceClient?> GetBlobService(string accountId)
{
_log.Info($"getting blob container (account_id: {accountId}");
var(accountName, accountKey) = await _storage.GetStorageAccountNameAndKey(accountId);
if (accountName == null)
{
_log.Error("Failed to get storage account name");
return(null);
}
var storageKeyCredential = new StorageSharedKeyCredential(accountName, accountKey);
var accountUrl = GetUrl(accountName);
return(new BlobServiceClient(accountUrl, storageKeyCredential));
}
public async Async.Task <HttpResponseData> NotOk(HttpRequestData request, Error error, string context, HttpStatusCode statusCode = HttpStatusCode.BadRequest)
{
var statusNum = (int)statusCode;
if (statusNum >= 400 && statusNum <= 599)
{
_log.Error($"request error - {context}: {error}");
var response = HttpResponseData.CreateResponse(request);
await response.WriteAsJsonAsync(error);
response.StatusCode = statusCode;
return(response);
}
throw new ArgumentOutOfRangeException($"status code {statusCode} - {statusNum} is not in the expected range [400; 599]");
}
private async Async.Task <bool> ScheduleWorkset(WorkSet workSet, Pool pool, long count)
{
if (!PoolStateHelper.Available.Contains(pool.State))
{
_logTracer.Info($"pool not available for work: {pool.Name} state: {pool.State}");
return(false);
}
for (var i = 0L; i < count; i++)
{
if (!await _poolOperations.ScheduleWorkset(pool, workSet))
{
_logTracer.Error($"unable to schedule workset. pool:{pool.Name} workset: {workSet}");
return(false);
}
}
return(true);
}
public async Task <bool> DeleteDisk(string resourceGroup, string name)
{
try {
_logTracer.Info($"deleting disks {resourceGroup} : {name}");
var disk = await _creds.GetResourceGroupResource().GetDiskAsync(name);
if (disk != null)
{
await disk.Value.DeleteAsync(WaitUntil.Started);
return(true);
}
} catch (Exception e) {
_logTracer.Error($"unable to delete disk: {name} {e.Message}");
_logTracer.Exception(e);
}
return(false);
}
public async Task <OneFuzzResultVoid> CreateVirtualNetwork(string resourceGroup, string name, string region, NetworkConfig networkConfig)
{
_logTracer.Info($"creating subnet - resource group:{resourceGroup} name:{name} region: {region}");
var virtualNetParam = new VirtualNetworkData {
Location = region,
};
virtualNetParam.AddressPrefixes.Add(networkConfig.AddressSpace);
virtualNetParam.Subnets.Add(new SubnetData {
Name = name,
AddressPrefix = networkConfig.Subnet
}
);
var onefuzzOwner = _context.ServiceConfiguration.OneFuzzOwner;
if (!string.IsNullOrEmpty(onefuzzOwner))
{
if (!virtualNetParam.Tags.TryAdd("OWNER", onefuzzOwner))
{
_logTracer.Warning($"Failed to add tag 'OWNER':{onefuzzOwner} to virtual network {resourceGroup}:{name}");
}
}
try {
await _creds.GetResourceGroupResource().GetVirtualNetworks().CreateOrUpdateAsync(
WaitUntil.Started,
name,
virtualNetParam
);
} catch (RequestFailedException ex) {
_logTracer.Error($"network creation failed: {name}:{region} {{error}}");
return(OneFuzzResultVoid.Error(
ErrorCode.UNABLE_TO_CREATE_NETWORK,
ex.ToString()
));
}
return(OneFuzzResultVoid.Ok);
}
public async Task <HttpResponseData> SaveToKeyvault([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/secrets/keyvault")] HttpRequestData req)
{
var s = await req.ReadAsStringAsync();
var secretData = JsonSerializer.Deserialize <SecretData <string> >(s !, EntityConverter.GetJsonSerializerOptions());
if (secretData is null)
{
_log.Error("Secret data is null");
return(req.CreateResponse(HttpStatusCode.BadRequest));
}
else
{
_log.Info($"Saving secret data in the keyvault");
var r = await _secretOps.SaveToKeyvault(secretData);
var addr = _secretOps.GetKeyvaultAddress();
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteAsJsonAsync(addr);
return(resp);
}
}
public async Task <HttpResponseData> Get([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/instance-config")] HttpRequestData req)
{
_log.Info("Fetching instance config");
var config = await _configOps.Fetch();
if (config is null)
{
_log.Error("Instance config is null");
Error err = new(ErrorCode.INVALID_REQUEST, new[] { "Instance config is null" });
var resp = req.CreateResponse(HttpStatusCode.InternalServerError);
await resp.WriteAsJsonAsync(err);
return(resp);
}
else
{
var str = EntityConverter.ToJsonString(config);
var resp = req.CreateResponse(HttpStatusCode.OK);
await resp.WriteStringAsync(str);
return(resp);
}
}
public async Async.Task SetSize(int size)
{
await Clear();
var i = 0;
while (i < size)
{
var r = await AddEntry();
if (r)
{
i++;
}
else
{
//TODO: retry after a delay ? I guess make a decision on this
//if we hit this error message... For now just log and move on to
//make it behave same as Python code.
_log.Error($"failed to add entry to shrink queue");
i++;
}
}
}
public async Task <bool> QueueObject <T>(string name, T obj, StorageType storageType, TimeSpan?visibilityTimeout = null, TimeSpan?timeToLive = null)
{
var queueClient = await GetQueueClient(name, storageType) ?? throw new Exception($"unable to queue object, no such queue: {name}");
try {
var serialized = JsonSerializer.Serialize(obj, EntityConverter.GetJsonSerializerOptions());
var res = await queueClient.SendMessageAsync(serialized, visibilityTimeout : visibilityTimeout, timeToLive);
if (res.GetRawResponse().IsError)
{
_log.Error($"Failed to send message {serialized} in queue {name} due to {res.GetRawResponse().ReasonPhrase}");
return(false);
}
else
{
return(true);
}
} catch (Exception ex) {
_log.Exception(ex, $"Failed to queue message in queue {name}");
return(false);
}
}
private async Async.Task <Error?> OnStateUpdate(Guid machineId, NodeStateUpdate ev)
{
var node = await _context.NodeOperations.GetByMachineId(machineId);
if (node is null)
{
_log.Warning($"unable to process state update event. machine_id:{machineId} state event:{ev}");
return(null);
}
if (ev.State == NodeState.Free)
{
if (node.ReimageRequested || node.DeleteRequested)
{
_log.Info($"stopping free node with reset flags: {machineId}");
await _context.NodeOperations.Stop(node);
return(null);
}
if (await _context.NodeOperations.CouldShrinkScaleset(node))
{
_log.Info($"stopping free node to resize scaleset: {machineId}");
await _context.NodeOperations.SetHalt(node);
return(null);
}
}
if (ev.State == NodeState.Init)
{
if (node.DeleteRequested)
{
_log.Info($"stopping node (init and delete_requested): {machineId}");
await _context.NodeOperations.Stop(node);
return(null);
}
// Don’t check reimage_requested, as nodes only send 'init' state once. If
// they send 'init' with reimage_requested, it's because the node was reimaged
// successfully.
node = node with {
ReimageRequested = false, InitializedAt = DateTimeOffset.UtcNow
};
await _context.NodeOperations.SetState(node, ev.State);
return(null);
}
_log.Info($"node state update: {machineId} from {node.State} to {ev.State}");
await _context.NodeOperations.SetState(node, ev.State);
if (ev.State == NodeState.Free)
{
_log.Info($"node now available for work: {machineId}");
}
else if (ev.State == NodeState.SettingUp)
{
if (ev.Data is NodeSettingUpEventData settingUpData)
{
if (!settingUpData.Tasks.Any())
{
return(new Error(ErrorCode.INVALID_REQUEST, Errors: new string[] {
$"setup without tasks. machine_id: {machineId}",
}));
}
foreach (var taskId in settingUpData.Tasks)
{
var task = await _context.TaskOperations.GetByTaskId(taskId);
if (task is null)
{
return(new Error(
ErrorCode.INVALID_REQUEST,
Errors: new string[] { $"unable to find task: {taskId}" }));
}
_log.Info($"node starting task. machine_id: {machineId} job_id: {task.JobId} task_id: {task.TaskId}");
// The task state may be `running` if it has `vm_count` > 1, and
// another node is concurrently executing the task. If so, leave
// the state as-is, to represent the max progress made.
//
// Other states we would want to preserve are excluded by the
// outermost conditional check.
if (task.State != TaskState.Running && task.State != TaskState.SettingUp)
{
await _context.TaskOperations.SetState(task, TaskState.SettingUp);
}
var nodeTask = new NodeTasks(
MachineId: machineId,
TaskId: task.TaskId,
State: NodeTaskState.SettingUp);
await _context.NodeTasksOperations.Replace(nodeTask);
}
}
}
else if (ev.State == NodeState.Done)
{
Error?error = null;
if (ev.Data is NodeDoneEventData doneData)
{
if (doneData.Error is not null)
{
var errorText = EntityConverter.ToJsonString(doneData);
error = new Error(ErrorCode.TASK_FAILED, Errors: new string[] { errorText });
_log.Error($"node 'done' with error: machine_id:{machineId}, data:{errorText}");
}
}
// if tasks are running on the node when it reports as Done
// those are stopped early
await _context.NodeOperations.MarkTasksStoppedEarly(node, error);
await _context.NodeOperations.ToReimage(node, done : true);
}
return(null);
}
//[Function("TimerProxy")]
public async Async.Task Run([TimerTrigger("1.00:00:00")] TimerInfo myTimer)
{
var proxies = await _proxYOperations.QueryAsync().ToListAsync();
foreach (var proxy in proxies)
{
if (VmStateHelper.Available.Contains(proxy.State))
{
// Note, outdated checked at the start, but set at the end of this loop.
// As this function is called via a timer, this works around a user
// requesting to use the proxy while this function is checking if it's
// out of date
if (proxy.Outdated)
{
await _proxYOperations.SetState(proxy, VmState.Stopping);
// If something is "wrong" with a proxy, delete & recreate it
}
else if (!_proxYOperations.IsAlive(proxy))
{
_logger.Error($"scaleset-proxy: alive check failed, stopping: {proxy.Region}");
await _proxYOperations.SetState(proxy, VmState.Stopping);
}
else
{
await _proxYOperations.SaveProxyConfig(proxy);
}
}
if (VmStateHelper.NeedsWork.Contains(proxy.State))
{
_logger.Error($"scaleset-proxy: update state. proxy:{proxy.Region} state:{proxy.State}");
await _proxYOperations.ProcessStateUpdate(proxy);
}
if (proxy.State != VmState.Stopped && _proxYOperations.IsOutdated(proxy))
{
await _proxYOperations.Replace(proxy with {
Outdated = true
});
}
}
// make sure there is a proxy for every currently active region
var regions = await _scalesetOperations.QueryAsync().Select(x => x.Region).ToHashSetAsync();
foreach (var region in regions)
{
var allOutdated = proxies.Where(x => x.Region == region).All(p => p.Outdated);
if (allOutdated)
{
await _proxYOperations.GetOrCreate(region);
_logger.Info($"Creating new proxy in region {region}");
}
// this is required in order to support upgrade from non-nsg to
// nsg enabled OneFuzz this will overwrite existing NSG
// assignment though. This behavior is acceptable at this point
// since we do not support bring your own NSG
if (await _nsg.GetNsg(region) != null)
{
var network = await Network.Create(region, _creds, _configOperations, _subnet);
var subnet = await network.GetSubnet();
var vnet = await network.GetVnet();
if (subnet != null && vnet != null)
{
var error = _nsg.AssociateSubnet(region, vnet, subnet);
if (error != null)
{
_logger.Error($"Failed to associate NSG and subnet due to {error} in region {region}");
}
}
}
// if there are NSGs with name same as the region that they are allocated
// and have no NIC associated with it then delete the NSG
await foreach (var nsg in _nsg.ListNsgs())
{
if (_nsg.OkToDelete(regions, nsg.Data.Location, nsg.Data.Name))
{
if (nsg.Data.NetworkInterfaces.Count == 0 && nsg.Data.Subnets.Count == 0)
{
await _nsg.StartDeleteNsg(nsg.Data.Name);
}
}
}
}
}
public async Async.Task <ResultVoid <TaskConfigError> > CheckConfig(TaskConfig config)
{
if (!Defs.TASK_DEFINITIONS.ContainsKey(config.Task.Type))
{
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError($"unsupported task type: {config.Task.Type}")));
}
if (config.Vm != null && config.Pool != null)
{
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError($"either the vm or pool must be specified, but not both")));
}
var definition = Defs.TASK_DEFINITIONS[config.Task.Type];
var r = await CheckContainers(definition, config);
if (!r.IsOk)
{
return(r);
}
if (definition.Features.Contains(TaskFeature.SupervisorExe) && config.Task.SupervisorExe == null)
{
var err = "missing supervisor_exe";
_logTracer.Error(err);
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError(err)));
}
if (definition.Features.Contains(TaskFeature.TargetMustUseInput) && !TargetUsesInput(config))
{
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError("{input} must be used in target_env or target_options")));
}
if (config.Vm != null)
{
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError("specifying task config vm is no longer supported")));
}
if (config.Pool == null)
{
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError("pool must be specified")));
}
if (!CheckVal(definition.Vm.Compare, definition.Vm.Value, config.Pool !.Count))
{
var err =
$"invalid vm count: expected {definition.Vm.Compare} {definition.Vm.Value}, got {config.Pool.Count}";
_logTracer.Error(err);
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError(err)));
}
var pool = await _context.PoolOperations.GetByName(config.Pool.PoolName);
if (!pool.IsOk)
{
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError($"invalid pool: {config.Pool.PoolName}")));
}
var checkTarget = await CheckTargetExe(config, definition);
if (!checkTarget.IsOk)
{
return(checkTarget);
}
if (definition.Features.Contains(TaskFeature.GeneratorExe))
{
var container = config.Containers !.First(x => x.Type == ContainerType.Tools);
if (config.Task.GeneratorExe == null)
{
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError($"generator_exe is not defined")));
}
var tool_paths = new[] { "{tools_dir}/", "{tools_dir}\\" };
foreach (var toolPath in tool_paths)
{
if (config.Task.GeneratorExe.StartsWith(toolPath))
{
var generator = config.Task.GeneratorExe.Replace(toolPath, "");
if (!await _containers.BlobExists(container.Name, generator, StorageType.Corpus))
{
var err =
$"generator_exe `{config.Task.GeneratorExe}` does not exist in the tools container `{container.Name}`";
_logTracer.Error(err);
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError(err)));
}
}
}
}
if (definition.Features.Contains(TaskFeature.StatsFile))
{
if (config.Task.StatsFile != null && config.Task.StatsFormat == null)
{
var err2 = "using a stats_file requires a stats_format";
_logTracer.Error(err2);
return(ResultVoid <TaskConfigError> .Error(new TaskConfigError(err2)));
}
}
return(ResultVoid <TaskConfigError> .Ok());
}