public async Async.Task <OneFuzzResultVoid> AddAutoScaleToVmss(Guid vmss, AutoscaleProfile autoScaleProfile)
{
_logTracer.Info($"Checking scaleset {vmss} for existing auto scale resource");
var existingAutoScaleResource = GetAutoscaleSettings(vmss);
if (!existingAutoScaleResource.IsOk)
{
return(OneFuzzResultVoid.Error(existingAutoScaleResource.ErrorV));
}
if (existingAutoScaleResource.OkV != null)
{
_logTracer.Warning($"Scaleset {vmss} already has auto scale resource");
}
var autoScaleResource = await CreateAutoScaleResourceFor(vmss, await _context.Creds.GetBaseRegion(), autoScaleProfile);
if (!autoScaleResource.IsOk)
{
return(OneFuzzResultVoid.Error(autoScaleResource.ErrorV));
}
var diagnosticsResource = await SetupAutoScaleDiagnostics(autoScaleResource.OkV.Id !, autoScaleResource.OkV.Data.Name, _context.LogAnalytics.GetWorkspaceId().ToString());
if (!diagnosticsResource.IsOk)
{
return(OneFuzzResultVoid.Error(diagnosticsResource.ErrorV));
}
return(OneFuzzResultVoid.Ok);
}
public async Async.Task <OneFuzzResultVoid> UpdateScaleInProtection(Guid name, Guid vmId, bool protectFromScaleIn)
{
var res = await GetInstanceVm(name, vmId);
if (!res.IsOk)
{
return(OneFuzzResultVoid.Error(res.ErrorV));
}
else
{
var instanceVm = res.OkV;
instanceVm.Data.ProtectionPolicy ??= new();
if (instanceVm.Data.ProtectionPolicy.ProtectFromScaleIn != protectFromScaleIn)
{
instanceVm.Data.ProtectionPolicy.ProtectFromScaleIn = protectFromScaleIn;
var vmCollection = GetVmssResource(name).GetVirtualMachineScaleSetVms();
try {
await vmCollection.CreateOrUpdateAsync(WaitUntil.Started, instanceVm.Data.InstanceId, instanceVm.Data);
return(OneFuzzResultVoid.Ok);
} catch {
var msg = $"unable to set protection policy on: {vmId}:{instanceVm.Id}";
return(OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, msg));
}
}
else
{
_log.Info($"scale in protection was already set to {protectFromScaleIn} on vm {vmId} for scaleset {name}");
return(OneFuzzResultVoid.Ok);
}
}
}
public async Async.Task <OneFuzzResultVoid> ResizeVmss(Guid name, long capacity)
{
var canUpdate = await CheckCanUpdate(name);
if (canUpdate.IsOk)
{
_log.Info($"updating VM count - name: {name} vm_count: {capacity}");
var scalesetResource = GetVmssResource(name);
var patch = new VirtualMachineScaleSetPatch();
patch.Sku.Capacity = capacity;
await scalesetResource.UpdateAsync(WaitUntil.Started, patch);
return(OneFuzzResultVoid.Ok);
}
else
{
return(OneFuzzResultVoid.Error(canUpdate.ErrorV));
}
}
public async Task <OneFuzzResultVoid> CreateVirtualNetwork(string resourceGroup, string name, string region, NetworkConfig networkConfig)
{
_logTracer.Info($"creating subnet - resource group:{resourceGroup} name:{name} region: {region}");
var virtualNetParam = new VirtualNetworkData {
Location = region,
};
virtualNetParam.AddressPrefixes.Add(networkConfig.AddressSpace);
virtualNetParam.Subnets.Add(new SubnetData {
Name = name,
AddressPrefix = networkConfig.Subnet
}
);
var onefuzzOwner = _context.ServiceConfiguration.OneFuzzOwner;
if (!string.IsNullOrEmpty(onefuzzOwner))
{
if (!virtualNetParam.Tags.TryAdd("OWNER", onefuzzOwner))
{
_logTracer.Warning($"Failed to add tag 'OWNER':{onefuzzOwner} to virtual network {resourceGroup}:{name}");
}
}
try {
await _creds.GetResourceGroupResource().GetVirtualNetworks().CreateOrUpdateAsync(
WaitUntil.Started,
name,
virtualNetParam
);
} catch (RequestFailedException ex) {
_logTracer.Error($"network creation failed: {name}:{region} {{error}}");
return(OneFuzzResultVoid.Error(
ErrorCode.UNABLE_TO_CREATE_NETWORK,
ex.ToString()
));
}
return(OneFuzzResultVoid.Ok);
}
public async Task <OneFuzzResultVoid> UpdateAutoscale(AutoscaleSettingData autoscale)
{
_logTracer.Info($"Updating auto scale resource: {autoscale.Name}");
try {
var newResource = await _context.Creds.GetResourceGroupResource().GetAutoscaleSettings().CreateOrUpdateAsync(
WaitUntil.Started,
autoscale.Name,
autoscale
);
_logTracer.Info($"Successfully updated auto scale resource: {autoscale.Name}");
} catch (RequestFailedException ex) {
_logTracer.Exception(ex);
return(OneFuzzResultVoid.Error(
ErrorCode.UNABLE_TO_UPDATE,
$"unable to update auto scale resource with name: {autoscale.Name} and profile: {JsonSerializer.Serialize(autoscale)}"
));
}
return(OneFuzzResultVoid.Ok);
}
public async Async.Task <OneFuzzResultVoid> UpdateScaleInProtection(Guid name, Guid vmId, bool protectFromScaleIn)
{
var res = await GetInstanceVm(name, vmId);
if (!res.IsOk)
{
return(OneFuzzResultVoid.Error(res.ErrorV));
}
else
{
VirtualMachineScaleSetVmProtectionPolicy newProtectionPolicy;
var instanceVm = res.OkV !;
if (instanceVm.Data.ProtectionPolicy is not null)
{
newProtectionPolicy = instanceVm.Data.ProtectionPolicy;
newProtectionPolicy.ProtectFromScaleIn = protectFromScaleIn;
}
else
{
newProtectionPolicy = new VirtualMachineScaleSetVmProtectionPolicy()
{
ProtectFromScaleIn = protectFromScaleIn
};
}
instanceVm.Data.ProtectionPolicy = newProtectionPolicy;
var scaleSet = GetVmssResource(name);
VirtualMachineScaleSetVmInstanceRequiredIds ids = new VirtualMachineScaleSetVmInstanceRequiredIds(new[] { instanceVm.Data.InstanceId });
var updateRes = await scaleSet.UpdateInstancesAsync(WaitUntil.Started, ids);
//TODO: finish this after UpdateInstance method is fixed
//https://github.com/Azure/azure-sdk-for-net/issues/28491
throw new NotImplementedException("Update instance does not work as expected. See https://github.com/Azure/azure-sdk-for-net/issues/28491");
}
}
public async Async.Task <OneFuzzResultVoid> UpdateExtensions(Guid name, IList <VirtualMachineScaleSetExtensionData> extensions)
{
var canUpdate = await CheckCanUpdate(name);
if (canUpdate.IsOk)
{
_log.Info($"updating VM extensions: {name}");
var res = GetVmssResource(name);
var patch = new VirtualMachineScaleSetPatch();
foreach (var ext in extensions)
{
patch.VirtualMachineProfile.ExtensionProfile.Extensions.Add(ext);
}
var _ = await res.UpdateAsync(WaitUntil.Started, patch);
_log.Info($"VM extensions updated: {name}");
return(OneFuzzResultVoid.Ok);
}
else
{
return(OneFuzzResultVoid.Error(canUpdate.ErrorV));
}
}
public async Async.Task <OneFuzzResultVoid> CreateVmss(
string location,
Guid name,
string vmSku,
long vmCount,
string image,
string networkId,
bool?spotInstance,
bool ephemeralOsDisks,
IList <VirtualMachineScaleSetExtensionData>?extensions,
string password,
string sshPublicKey,
IDictionary <string, string> tags)
{
var vmss = await GetVmss(name);
if (vmss is not null)
{
return(OneFuzzResultVoid.Ok);
}
_log.Info($"creating VM name: {name}, vm_sku: {vmSku}, vm_count: {vmCount}, image: {image}, subnet: {networkId}, spot_instance: {spotInstance}");
var getOsResult = await _imageOps.GetOs(location, image);
if (!getOsResult.IsOk)
{
return(getOsResult.ErrorV);
}
var vmssData = new VirtualMachineScaleSetData(location)
{
DoNotRunExtensionsOnOverprovisionedVms = false,
Sku = new ComputeSku()
{
Name = vmSku
},
Overprovision = false,
SinglePlacementGroup = false,
UpgradePolicy = new UpgradePolicy()
{
Mode = UpgradeMode.Manual
},
Identity = new ManagedServiceIdentity(managedServiceIdentityType: ManagedServiceIdentityType.UserAssigned),
};
vmssData.Identity.UserAssignedIdentities.Add(_creds.GetScalesetIdentityResourcePath(), new UserAssignedIdentity());
vmssData.VirtualMachineProfile = new VirtualMachineScaleSetVmProfile()
{
Priority = VirtualMachinePriorityTypes.Regular
};
var imageRef = new ImageReference();
if (image.StartsWith('/'))
{
imageRef.Id = image;
}
else
{
var info = IImageOperations.GetImageInfo(image);
imageRef.Publisher = info.Publisher;
imageRef.Offer = info.Offer;
imageRef.Sku = info.Sku;
imageRef.Version = info.Version;
}
vmssData.VirtualMachineProfile.StorageProfile = new VirtualMachineScaleSetStorageProfile()
{
ImageReference = imageRef
};
vmssData.VirtualMachineProfile.OSProfile = new VirtualMachineScaleSetOSProfile()
{
ComputerNamePrefix = "node", AdminUsername = "******"
};
var networkConfiguration = new VirtualMachineScaleSetNetworkConfiguration("onefuzz-nic")
{
Primary = true
};
var ipConfig = new VirtualMachineScaleSetIPConfiguration("onefuzz-ip-config");
ipConfig.SubnetId = new ResourceIdentifier(networkId);
networkConfiguration.IPConfigurations.Add(ipConfig);
vmssData.VirtualMachineProfile.NetworkProfile = new VirtualMachineScaleSetNetworkProfile();
vmssData.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations.Add(networkConfiguration);
if (extensions is not null)
{
vmssData.VirtualMachineProfile.ExtensionProfile = new VirtualMachineScaleSetExtensionProfile();
foreach (var e in extensions)
{
vmssData.VirtualMachineProfile.ExtensionProfile.Extensions.Add(e);
}
}
switch (getOsResult.OkV)
{
case Os.Windows:
vmssData.VirtualMachineProfile.OSProfile.AdminPassword = password;
break;
case Os.Linux:
vmssData.VirtualMachineProfile.OSProfile.LinuxConfiguration = new LinuxConfiguration();
vmssData.VirtualMachineProfile.OSProfile.LinuxConfiguration.DisablePasswordAuthentication = true;
var i = new SshPublicKeyInfo()
{
KeyData = sshPublicKey, Path = "/home/onefuzz/.ssh/authorized_keys"
};
vmssData.VirtualMachineProfile.OSProfile.LinuxConfiguration.SshPublicKeys.Add(i);
break;
default:
return(OneFuzzResultVoid.Error(ErrorCode.INVALID_CONFIGURATION, $"unhandled OS: {getOsResult.OkV} in image: {image}"));
}
if (ephemeralOsDisks)
{
vmssData.VirtualMachineProfile.StorageProfile.OSDisk = new VirtualMachineScaleSetOSDisk(DiskCreateOptionTypes.FromImage);
vmssData.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings = new DiffDiskSettings();
vmssData.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings.Option = DiffDiskOptions.Local;
vmssData.VirtualMachineProfile.StorageProfile.OSDisk.Caching = CachingTypes.ReadOnly;
}
if (spotInstance.HasValue && spotInstance.Value)
{
// Setting max price to -1 means it won't be evicted because of
// price.
//
// https://docs.microsoft.com/en-us/azure/
// virtual-machine-scale-sets/use-spot#resource-manager-templates
vmssData.VirtualMachineProfile.EvictionPolicy = VirtualMachineEvictionPolicyTypes.Deallocate;
vmssData.VirtualMachineProfile.Priority = VirtualMachinePriorityTypes.Spot;
vmssData.VirtualMachineProfile.BillingMaxPrice = 1.0;
}
foreach (var tag in tags)
{
vmssData.Tags.Add(tag);
}
if (_serviceConfig.OneFuzzOwner is not null)
{
vmssData.Tags.Add("OWNER", _serviceConfig.OneFuzzOwner);
}
try {
var rg = _creds.GetResourceGroupResource();
var createUpdate = await rg.GetVirtualMachineScaleSets().CreateOrUpdateAsync(WaitUntil.Started, name.ToString(), vmssData);
if (createUpdate.GetRawResponse().IsError)
{
var msg = $"Failed to create new scaleset due to {createUpdate.GetRawResponse().ReasonPhrase}";
_log.Error(msg);
return(OneFuzzResultVoid.Error(ErrorCode.VM_CREATE_FAILED, new[] { msg }));
}
else
{
return(OneFuzzResultVoid.Ok);
}
} catch (Exception ex) {
_log.Exception(ex);
return(OneFuzzResultVoid.Error(ErrorCode.VM_CREATE_FAILED, new[] { ex.Message }));
}
}
public async Task <OneFuzzResultVoid> CreatePublicNic(string resourceGroup, string name, string region, Nsg?nsg)
{
_logTracer.Info($"creating nic for {resourceGroup}:{name} in {region}");
var network = await Network.Create(region, _context);
var subnetId = await network.GetId();
if (subnetId is null)
{
await network.Create();
return(OneFuzzResultVoid.Ok);
}
if (nsg != null)
{
var subnet = await network.GetSubnet();
if (subnet != null && subnet.Data.NetworkSecurityGroup == null)
{
var vnet = await network.GetVnet();
var result = await _context.NsgOperations.AssociateSubnet(nsg.Name, vnet !, subnet);
if (!result.IsOk)
{
return(OneFuzzResultVoid.Error(result.ErrorV));
}
return(OneFuzzResultVoid.Ok);
}
}
var ip = await GetIp(resourceGroup, name);
if (ip == null)
{
await CreateIp(resourceGroup, name, region);
return(OneFuzzResultVoid.Ok);
}
var networkInterface = new NetworkInterfaceData {
Location = region,
};
networkInterface.IPConfigurations.Add(new NetworkInterfaceIPConfigurationData {
Name = "myIPConfig",
PublicIPAddress = ip?.Data,
Subnet = new SubnetData {
Id = subnetId
}
}
);
var onefuzzOwner = _context.ServiceConfiguration.OneFuzzOwner;
if (!string.IsNullOrEmpty(onefuzzOwner))
{
if (!networkInterface.Tags.TryAdd("OWNER", onefuzzOwner))
{
_logTracer.Warning($"Failed to add tag 'OWNER':{onefuzzOwner} to nic {resourceGroup}:{name}");
}
}
try {
await _context.Creds.GetResourceGroupResource().GetNetworkInterfaces().CreateOrUpdateAsync(
WaitUntil.Started,
name,
networkInterface
);
} catch (RequestFailedException ex) {
if (!ex.ToString().Contains("RetryableError"))
{
return(OneFuzzResultVoid.Error(
ErrorCode.VM_CREATE_FAILED,
$"unable to create nic: {ex}"
));
}
}
return(OneFuzzResultVoid.Ok);
}
async Task <OneFuzzResultVoid> CreateVm(
string name,
string location,
string vmSku,
string image,
string password,
string sshPublicKey,
Nsg?nsg,
IDictionary <string, string>?tags
)
{
var resourceGroup = _context.Creds.GetBaseResourceGroup();
_logTracer.Info($"creating vm {resourceGroup}:{location}:{name}");
var nic = await _context.IpOperations.GetPublicNic(resourceGroup, name);
if (nic == null)
{
var result = await _context.IpOperations.CreatePublicNic(resourceGroup, name, location, nsg);
if (!result.IsOk)
{
return(result);
}
_logTracer.Info("waiting on nic creation");
return(OneFuzzResultVoid.Ok);
}
// when public nic is created, VNET must exist at that point
// this is logic of get_public_nic function
if (nsg != null)
{
var result = await _context.NsgOperations.AssociateNic(nsg, nic);
if (!result.IsOk)
{
return(result);
}
}
var vmParams = new VirtualMachineData(location)
{
OSProfile = new OSProfile {
ComputerName = "node",
AdminUsername = "******",
},
HardwareProfile = new HardwareProfile {
VmSize = vmSku,
},
StorageProfile = new StorageProfile {
ImageReference = GenerateImageReference(image),
},
NetworkProfile = new NetworkProfile(),
};
vmParams.NetworkProfile.NetworkInterfaces.Add(new NetworkInterfaceReference {
Id = nic.Id
});
var imageOs = await _context.ImageOperations.GetOs(location, image);
if (!imageOs.IsOk)
{
return(OneFuzzResultVoid.Error(imageOs.ErrorV));
}
switch (imageOs.OkV)
{
case Os.Windows: {
vmParams.OSProfile.AdminPassword = password;
break;
}
case Os.Linux: {
vmParams.OSProfile.LinuxConfiguration = new LinuxConfiguration {
DisablePasswordAuthentication = true,
};
vmParams.OSProfile.LinuxConfiguration.SshPublicKeys.Add(
new SshPublicKeyInfo {
Path = "/home/onefuzz/.ssh/authorized_keys",
KeyData = sshPublicKey
}
);
break;
}
default: throw new NotSupportedException($"No support for OS type: {imageOs.OkV}");
}
var onefuzzOwner = _context.ServiceConfiguration.OneFuzzOwner;
if (!string.IsNullOrEmpty(onefuzzOwner))
{
vmParams.Tags.Add("OWNER", onefuzzOwner);
}
else
{
tags?.ToList()
.ForEach(kvp => {
if (!vmParams.Tags.TryAdd(kvp.Key, kvp.Value))
{
_logTracer.Warning($"Failed to add tag {kvp.Key}:{kvp.Value} to vm {name}");
}
});
}
try {
await _context.Creds.GetResourceGroupResource().GetVirtualMachines().CreateOrUpdateAsync(
WaitUntil.Started,
name,
vmParams
);
} catch (RequestFailedException ex) {
if (ex.ErrorCode == "ResourceNotFound" && ex.Message.Contains("The request failed due to conflict with a concurrent request"))
{
// _logTracer.Debug($"create VM had conflicts with concurrent request, ignoring {ex.ToString()}");
return(OneFuzzResultVoid.Ok);
}
return(OneFuzzResultVoid.Error(
ErrorCode.VM_CREATE_FAILED,
ex.ToString()
));
}
return(OneFuzzResultVoid.Ok);
}