public void willCancelBIOSWrite(TimeSpan delay)
{
using (bladeDirectorDebugServices svc = new bladeDirectorDebugServices(basicBladeTests.WCFPath, basicBladeTests.WebURI))
{
machinePools.bladeDirectorURL = svc.servicesURL;
string hostIP = "1.1.1.1";
bladeSpec spec = svc.svcDebug.createBladeSpec("172.17.129.131", "192.168.129.131", "172.17.2.131", 1234, false, VMDeployStatus.notBeingDeployed, " ... ", "idk", "box", bladeLockType.lockAll, bladeLockType.lockAll);
svc.svcDebug.initWithBladesFromBladeSpec(new[] { spec }, false, NASFaultInjectionPolicy.retunSuccessful);
resultAndBladeName res = svc.svcDebug._RequestAnySingleNode(hostIP);
Assert.AreEqual(resultCode.success, res.result.code);
string ourBlade = ((resultAndBladeName)testUtils.waitForSuccess(svc, res, TimeSpan.FromSeconds(30))).bladeName;
// Start a BIOS read
resultAndWaitToken readRes = svc.svcDebug._rebootAndStartReadingBIOSConfiguration(hostIP, ourBlade);
Assert.AreEqual(resultCode.pending, readRes.result.code);
// Wait a while to see if we get somewhere where it is impossible to free without a hueg delay
Thread.Sleep(delay);
// Then free the blade. The BIOS operation should be cancelled before it soaks up all the ten minutes of time.
resultAndWaitToken releaseRes = svc.svcDebug._ReleaseBladeOrVM(hostIP, ourBlade, false);
testUtils.waitForSuccess(svc, releaseRes, TimeSpan.FromMinutes(1));
// And it should be no longer getting.
resultAndWaitToken bladeState = svc.svc.getProgress(readRes.waitToken);
Assert.AreNotEqual(resultCode.pending, bladeState.result.code);
// It no longer be allocated to us, since we released it earlier.
GetBladeStatusResult ownershipRes = svc.svc.GetBladeStatus(ourBlade);
Assert.AreEqual(GetBladeStatusResult.unused, ownershipRes);
}
}
public void willSetBIOS()
{
string testBiosXML = Properties.Resources.testBIOS;
using (bladeDirectorDebugServices svc = new bladeDirectorDebugServices(basicBladeTests.WCFPath, basicBladeTests.WebURI))
{
machinePools.bladeDirectorURL = svc.servicesURL;
string hostip = "1.2.3.4";
// We will be using this blade for our tests.
bladeSpec spec = svc.svcDebug.createBladeSpec("172.17.129.131", "192.168.129.131", "172.17.2.131", 1234, false, VMDeployStatus.notBeingDeployed, " ... ", "idk", "box", bladeLockType.lockAll, bladeLockType.lockAll);
svc.svcDebug.initWithBladesFromBladeSpec(new[] { spec }, false, NASFaultInjectionPolicy.retunSuccessful);
resultAndBladeName res = svc.svcDebug._RequestAnySingleNode(hostip);
Assert.AreEqual(resultCode.success, res.result.code);
// Write the new BIOS. This configuration has the 'boot state' of the numlock key set to 'off', so we can
// read that back after writing to ensure that the BIOS write succeeds.
var resultAndReadBack = writeBIOSAndReadBack(svc, hostip, res.bladeName, testBiosXML);
Assert.IsTrue(resultAndReadBack.BIOSConfig.Contains("<Section name=\"NumLock\">Off</Section>"));
// Now we can modify the BIOS slightly, to set the numlock boot state to 'on', and again write/readback, and see
// if the change was carried out.
testBiosXML = testBiosXML.Replace("<Section name=\"NumLock\">Off</Section>", "<Section name=\"NumLock\">On</Section>");
resultAndReadBack = writeBIOSAndReadBack(svc, hostip, res.bladeName, testBiosXML);
Assert.IsTrue(resultAndReadBack.BIOSConfig.Contains("<Section name=\"NumLock\">On</Section>"));
resultAndWaitToken relRes = svc.svcDebug._ReleaseBladeOrVM(hostip, res.bladeName, false);
while (relRes.result.code == resultCode.pending)
{
Thread.Sleep(TimeSpan.FromSeconds(3));
relRes = svc.svc.getProgress(relRes.waitToken);
}
Assert.AreEqual(resultCode.success, relRes.result.code);
}
}
public hypervisorCollection <hypSpec_iLo> requestAsManyHypervisorsAsPossible(string snapshotName)
{
initialiseIfNeeded();
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
int nodeCount = director.svc.getAllBladeIP().Length;
hypervisorCollection <hypSpec_iLo> toRet = new hypervisorCollection <hypSpec_iLo>();
DateTime deadline = DateTime.Now + TimeSpan.FromMinutes(60);
try
{
for (int i = 0; i < nodeCount; i++)
{
resultAndBladeName res = director.svc.RequestAnySingleNode();
resultAndBladeName progress = director.waitForSuccess(res, deadline - DateTime.Now);
bladeSpec bladeConfig = director.svc.getBladeByIP_withoutLocking(progress.bladeName);
resultAndWaitToken snapRes = director.svc.selectSnapshotForBladeOrVM(progress.bladeName, snapshotName);
director.waitForSuccess(snapRes, TimeSpan.FromMinutes(3));
snapshotDetails snapshot = director.svc.getCurrentSnapshotDetails(progress.bladeName);
userDesc cred = bladeConfig.credentials.First();
NASParams nas = director.svc.getNASParams();
hypSpec_iLo spec = new hypSpec_iLo(
bladeConfig.bladeIP, cred.username, cred.password,
bladeConfig.iLOIP, bladeConfig.iLoUsername, bladeConfig.iLoPassword,
nas.IP, nas.username, nas.password,
snapshot.friendlyName, snapshot.path,
bladeConfig.kernelDebugPort, bladeConfig.kernelDebugKey
);
ipUtils.ensurePortIsFree(bladeConfig.kernelDebugPort);
bladeDirectedHypervisor_iLo newHyp = new bladeDirectedHypervisor_iLo(spec);
newHyp.setDisposalCallback(onDestruction);
if (!toRet.TryAdd(bladeConfig.bladeIP, newHyp))
{
throw new Exception();
}
}
return(toRet);
}
catch (Exception)
{
toRet.Dispose();
throw;
}
}
}
public void reportsVMCreationFailureNeatly()
{
string hostIP = "1.1.1.1";
using (bladeDirectorDebugServices uut = new bladeDirectorDebugServices(basicBladeTests.WCFPath, new[] { "1.1.1.1", "2.2.2.2", "3.3.3.3" }))
{
VMHardwareSpec hwSpec = new VMHardwareSpec()
{
memoryMB = 2344, cpuCount = 2
};
VMSoftwareSpec swSpec = new VMSoftwareSpec();
uut.svcDebug.initWithBladesFromIPList(new[] { "172.17.129.131", "172.17.129.130" }, true, NASFaultInjectionPolicy.failSnapshotDeletionOnFirstSnapshot);
uut.svcDebug._setExecutionResultsIfMocked(mockedExecutionResponses.successful);
resultAndBladeName allocRes;
allocRes = uut.svcDebug._requestAnySingleVM(hostIP, hwSpec, swSpec);
Assert.AreEqual(resultCode.pending, allocRes.result.code);
waitToken waitTok = allocRes.waitToken;
while (true)
{
Thread.Sleep(TimeSpan.FromSeconds(1));
resultAndBladeName progress = (resultAndBladeName)uut.svc.getProgress(waitTok);
waitTok = progress.waitToken;
if (progress.result.code == resultCode.pending)
{
continue;
}
Assert.AreEqual(resultCode.genericFail, progress.result.code);
Assert.AreEqual("172.17.158.1", progress.bladeName);
break;
}
// OK, our allocation failed. Try allocating a second VM - this one should succeed.
allocRes = uut.svcDebug._requestAnySingleVM(hostIP, hwSpec, swSpec);
Assert.AreEqual(resultCode.pending, allocRes.result.code);
allocRes = (resultAndBladeName)testUtils.waitForSuccess(uut, allocRes, TimeSpan.FromMinutes(11));
Assert.AreNotEqual("172.17.158.1", allocRes.bladeName);
Assert.AreNotEqual("172.17.158.1", allocRes.bladeName);
vmSpec VMConfig = uut.svc.getVMByIP_withoutLocking(allocRes.bladeName);
Assert.AreNotEqual("00:50:56:00:30:01", VMConfig.eth0MAC);
Assert.AreNotEqual("00:50:56:01:30:01", VMConfig.eth1MAC);
}
}
public void willTimeoutOnNoKeepalives()
{
using (bladeDirectorDebugServices uut = new bladeDirectorDebugServices(WCFPath, new[] { "1.1.1.1" }))
{
uut.svcDebug.setKeepAliveTimeout(10);
string hostip = "192.168.1.1";
resultAndWaitToken resp = uut.svcDebug._RequestAnySingleNode(hostip);
resultAndBladeName resWithName = (resultAndBladeName)resp;
Assert.AreEqual(resultCode.success, resp.result.code);
Assert.AreEqual("1.1.1.1", resWithName.bladeName);
Assert.AreEqual(GetBladeStatusResult.yours, uut.svcDebug._GetBladeStatus(hostip, resWithName.bladeName));
Thread.Sleep(TimeSpan.FromSeconds(11));
Assert.AreEqual(GetBladeStatusResult.unused, uut.svcDebug._GetBladeStatus(hostip, resWithName.bladeName));
}
}
public void willEnforceSharingWithVMs()
{
using (bladeDirectorDebugServices uut = new bladeDirectorDebugServices(WCFPath, new[] { "1.1.1.1", "1.1.1.2" }))
{
string hostipA = "192.168.1.1";
string hostipB = "192.168.1.2";
// Host A requests all the VMs
vmHWAndSWSpec toAlloc = new vmHWAndSWSpec(
new VMHardwareSpec()
{
memoryMB = 4, cpuCount = 1
},
new VMSoftwareSpec());
resultAndBladeName[] intialReqs = uut.svcDebug._requestAsManyVMAsPossible(hostipA, toAlloc.hw, toAlloc.sw);
for (int index = 0; index < intialReqs.Length; index++)
{
intialReqs[index] = (resultAndBladeName)testUtils.waitForSuccess(uut, intialReqs[index], TimeSpan.FromMinutes(1));
}
// Now B requests a VM
resultAndBladeName reqFromB = uut.svcDebug._requestAnySingleVM(hostipB, toAlloc.hw, toAlloc.sw);
Assert.AreEqual(resultCode.pending, reqFromB.result.code);
// A should be asked to release a single VM, and does as it is told
int releaseRequestCount = 0;
foreach (resultAndBladeName req in intialReqs)
{
Debug.WriteLine(uut.svcDebug._GetVMStatus(hostipA, req.bladeName));
if (uut.svcDebug._GetVMStatus(hostipA, req.bladeName) == GetBladeStatusResult.releasePending)
{
resultAndWaitToken relWait = uut.svcDebug._ReleaseBladeOrVM(hostipA, req.bladeName, false);
testUtils.waitForSuccess(uut, relWait, TimeSpan.FromSeconds(30));
releaseRequestCount++;
}
}
Assert.AreEqual(1, releaseRequestCount);
// Now, host B should have one VM.
resultAndWaitToken newVM = testUtils.waitForSuccess(uut, reqFromB, TimeSpan.FromSeconds(30));
Assert.AreEqual(resultCode.success, newVM.result.code);
}
}
private static resultAndBladeName[] doAllocation(bladeDirectorDebugServices uut, string hostIP, vmHWAndSWSpec[] specs)
{
resultAndBladeName[] allocRes = new resultAndBladeName[specs.Length];
for (int i = 0; i < specs.Length; i++)
{
allocRes[i] = uut.svcDebug._requestAnySingleVM(hostIP, specs[i].hw, specs[i].sw);
Assert.AreEqual(resultCode.pending, allocRes[i].result.code);
}
// Wait until all the allocation operations are complete
for (int i = 0; i < specs.Length; i++)
{
allocRes[i] = (resultAndBladeName)testUtils.waitForSuccess(uut, allocRes[i], TimeSpan.FromMinutes(15));
}
return(allocRes);
}
public static resultAndBladeName startAsyncVMAllocationForTest(bladeDirectorDebugServices uut, string hostIP)
{
uut.svcDebug._setExecutionResultsIfMocked(mockedExecutionResponses.successful);
VMHardwareSpec hwspec = new VMHardwareSpec
{
cpuCount = 1,
memoryMB = 1024 * 3
};
VMSoftwareSpec swspec = new VMSoftwareSpec();
resultAndBladeName allocRes = uut.svcDebug._requestAnySingleVM(hostIP, hwspec, swspec);
if (allocRes.result.code != resultCode.pending && allocRes.result.code != resultCode.success)
{
Assert.Fail("unexpected status: " + allocRes.result.code.ToString());
}
return(allocRes);
}
public void canAllocateVMMocked()
{
string hostIP = "1.1.1.1";
using (bladeDirectorDebugServices svc = new bladeDirectorDebugServices(basicBladeTests.WCFPath, new[] { "1.1.1.1", "2.2.2.2", "3.3.3.3" }))
{
VMHardwareSpec hwSpec = new VMHardwareSpec()
{
memoryMB = 2344, cpuCount = 2
};
VMSoftwareSpec swSpec = new VMSoftwareSpec();
resultAndBladeName[] allocRes = doVMAllocationsForTest(svc, hostIP, new[] { new vmHWAndSWSpec(hwSpec, swSpec) });
Assert.AreEqual(1, allocRes.Length);
resultAndBladeName allocatedBlade = allocRes[0];
// TODO: check nas events
mockedCall[] nasEvents = svc.svcDebug._getNASEventsIfMocked();
foreach (mockedCall call in nasEvents)
{
Debug.WriteLine(call.functionName + " " + call.message);
}
// TODO: check executions all happened okay
// This blade should become a VM server
GetBladeStatusResult allocated1 = svc.svcDebug._GetBladeStatus(hostIP, "172.17.129.130");
Assert.AreEqual(allocated1, GetBladeStatusResult.notYours);
// And there should now be one VM allocated to us at present.
vmSpec VMConfig = svc.svc.getVMByIP_withoutLocking(allocatedBlade.bladeName);
Assert.AreEqual("172.17.158.1", allocatedBlade.bladeName);
Assert.AreEqual("VM_30_01", VMConfig.friendlyName);
Assert.AreEqual("172.17.158.1", VMConfig.VMIP);
Assert.AreEqual("10.0.158.1", VMConfig.iscsiIP);
Assert.AreEqual("00:50:56:00:30:01", VMConfig.eth0MAC);
Assert.AreEqual("00:50:56:01:30:01", VMConfig.eth1MAC);
Assert.AreEqual(2344, VMConfig.memoryMB);
Assert.AreEqual(2, VMConfig.cpuCount);
Assert.AreEqual(hostIP, VMConfig.currentOwner);
}
}
public void willProvisionBlade()
{
using (bladeDirectorDebugServices svc = new bladeDirectorDebugServices(basicBladeTests.WCFPath, basicBladeTests.WebURI))
{
machinePools.bladeDirectorURL = svc.servicesURL;
string hostip = "1.2.3.4";
//string debuggerHost = testUtils.getBestRouteTo(IPAddress.Parse("172.17.129.131")).ToString();
// We will be using this blade for our tests.
bladeSpec spec = svc.svcDebug.createBladeSpecForXDLNode(31, "xdl.hacks.the.planet", bladeLockType.lockAll, bladeLockType.lockAll);
spec.friendlyName = "newBlade";
svc.svcDebug.initWithBladesFromBladeSpec(new[] { spec }, false, NASFaultInjectionPolicy.retunSuccessful);
resultAndBladeName res = svc.svcDebug._RequestAnySingleNode(hostip);
testUtils.waitForSuccess(svc, res, TimeSpan.FromMinutes(15));
string bladeName = res.bladeName;
resultAndWaitToken res2 = svc.svcDebug._selectSnapshotForBladeOrVM(hostip, bladeName, "discord");
testUtils.waitForSuccess(svc, res2, TimeSpan.FromMinutes(30));
// Okay, we have our blade allocated now.
bladeSpec createdBlade = svc.svc.getBladeByIP_withoutLocking(bladeName);
snapshotDetails snap = svc.svc.getCurrentSnapshotDetails(bladeName);
NASParams nas = svc.svc.getNASParams();
using (hypervisor foo = utils.createHypForBlade(createdBlade, snap, nas))
{
foo.powerOn(new cancellableDateTime(TimeSpan.FromMinutes(5)));
// Check that debugging has been provisioned correctly
executionResult bcdEditRes = foo.startExecutable("bcdedit", "/dbgsettings");
try
{
Assert.AreEqual(0, bcdEditRes.resultCode);
Assert.IsTrue(Regex.IsMatch(bcdEditRes.stdout, "key\\s*xdl.hacks.the.planet"), "bcdedit did not match regex for debug key");
Assert.IsTrue(Regex.IsMatch(bcdEditRes.stdout, "debugtype\\s*NET"), "bcdedit did not match regex for debug type");
Assert.IsTrue(Regex.IsMatch(bcdEditRes.stdout, "hostip\\s*1.2.3.4"), "bcdedit did not match regex for debug host");
Assert.IsTrue(Regex.IsMatch(bcdEditRes.stdout, "port\\s*53101"), "bcdedit did not match regex for debug port");
}
catch (AssertFailedException)
{
Debug.WriteLine("return code " + bcdEditRes.resultCode);
Debug.WriteLine("stdout " + bcdEditRes.stdout);
Debug.WriteLine("stderr " + bcdEditRes.stderr);
}
executionResult getNameRes = foo.startExecutable("echo %COMPUTERNAME%", "");
try
{
Assert.AreEqual(0, getNameRes.resultCode);
Assert.AreSame(getNameRes.stdout.ToLower(), "newBlade".Trim().ToLower(), "machine name was incorrect");
}
catch (AssertFailedException)
{
Debug.WriteLine("return code " + bcdEditRes.resultCode);
Debug.WriteLine("stdout " + bcdEditRes.stdout);
Debug.WriteLine("stderr " + bcdEditRes.stderr);
}
}
}
}
public void willProvisionVM()
{
using (bladeDirectorDebugServices svc = new bladeDirectorDebugServices(basicBladeTests.WCFPath, basicBladeTests.WebURI))
{
machinePools.bladeDirectorURL = svc.servicesURL;
string hostip = "1.2.3.4";
// We will be using this blade for our tests.
bladeSpec spec = svc.svcDebug.createBladeSpecForXDLNode(31, "xdl.hacks.the.planet", bladeLockType.lockAll, bladeLockType.lockAll);
svc.svcDebug.initWithBladesFromBladeSpec(new[] { spec }, false, NASFaultInjectionPolicy.retunSuccessful);
string debuggerHost = ipUtils.getBestRouteTo(IPAddress.Parse(spec.bladeIP)).ToString();
VMSoftwareSpec sw = new VMSoftwareSpec()
{
debuggerHost = debuggerHost,
debuggerKey = "a.b.c.d",
debuggerPort = 60234
};
VMHardwareSpec hw = new VMHardwareSpec()
{
cpuCount = 1, memoryMB = 4096
};
resultAndBladeName res = svc.svcDebug._requestAnySingleVM(hostip, hw, sw);
testUtils.waitForSuccess(svc, res, TimeSpan.FromMinutes(15));
string VMName = res.bladeName;
// Okay, we have our new VM allocated now.
vmSpec createdBlade = svc.svc.getVMByIP_withoutLocking(VMName);
bladeSpec parentBlade = svc.svc.getBladeByIP_withoutLocking(createdBlade.parentBladeIP);
snapshotDetails snap = svc.svc.getCurrentSnapshotDetails(VMName);
NASParams nas = svc.svc.getNASParams();
using (hypervisor_vmware_FreeNAS hyp = utils.createHypForVM(createdBlade, parentBlade, snap, nas))
{
hyp.powerOn(new cancellableDateTime(TimeSpan.FromMinutes(2)));
// Check that debugging has been provisioned correctly
executionResult bcdEditRes = hyp.startExecutable("bcdedit", "/dbgsettings");
try
{
Assert.AreEqual(0, bcdEditRes.resultCode);
Assert.IsTrue(Regex.IsMatch(bcdEditRes.stdout, "key\\s*a.b.c.d"));
Assert.IsTrue(Regex.IsMatch(bcdEditRes.stdout, "debugtype\\s*NET"));
Assert.IsTrue(Regex.IsMatch(bcdEditRes.stdout, "hostip\\s*" + hostip));
Assert.IsTrue(Regex.IsMatch(bcdEditRes.stdout, "port\\s*60234"));
}
catch (AssertFailedException)
{
Debug.WriteLine("bcdedit stdout " + bcdEditRes.stdout);
Debug.WriteLine("bcdedit stderr " + bcdEditRes.stderr);
throw;
}
executionResult wmicRes = hyp.startExecutable("wmic", "computersystem get totalPhysicalMemory,name,numberOfLogicalProcessors /format:value");
try
{
// We expect an response similar to:
//
// Name=ALIZANALYSIS
// NumberOfLogicalProcessors=8
// TotalPhysicalMemory=17119825920
//
Assert.AreEqual(0, wmicRes.resultCode);
string[] lines = wmicRes.stdout.Split(new[] { '\n', 'r' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string line in lines)
{
if (line.Trim().Length == 0)
{
continue;
}
string[] parts = line.Split('=');
string name = parts[0].ToLower().Trim();
string value = parts[1].ToLower().Trim();
switch (name)
{
case "name":
Assert.AreEqual("VM_31_01", value, "Name is incorrect");
break;
case "NumberOfLogicalProcessors":
Assert.AreEqual("1", value, "CPU count is incorrect");
break;
case "TotalPhysicalMemory":
Assert.AreEqual((4096L * 1024L * 1024L).ToString(), value, "RAM size is incorrect");
break;
default:
break;
}
}
}
catch (AssertFailedException)
{
Debug.WriteLine("WMIC reported stdout '" + wmicRes.stdout + "'");
Debug.WriteLine("WMIC reported stderr '" + wmicRes.stderr + "'");
}
}
}
}
public hypervisorCollection <hypSpec_vmware> requestVMs(VMSpec[] specs, bool allowPartialAllocation = false)
{
initialiseIfNeeded();
Stopwatch allocWatch = new Stopwatch();
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
DateTime deadline = DateTime.Now + TimeSpan.FromMinutes(30);
resultAndBladeName[] results = new resultAndBladeName[specs.Length];
for (int n = 0; n < specs.Length; n++)
{
results[n] = director.svc.RequestAnySingleVM(specs[n].hw, specs[n].sw);
if (results[n].result.code == resultCode.bladeQueueFull)
{
if (allowPartialAllocation)
{
results[n] = null;
break;
}
}
if (results[n].result.code != resultCode.success &&
results[n].result.code != resultCode.pending)
{
throw new bladeAllocationException(results[n].result.code);
}
}
hypervisorCollection <hypSpec_vmware> toRet = new hypervisorCollection <hypSpec_vmware>();
try
{
foreach (resultAndBladeName res in results)
{
if (res == null)
{
continue;
}
resultAndBladeName progress = director.waitForSuccessWithoutThrowing(res, deadline - DateTime.Now);
if (progress == null)
{
throw new TimeoutException();
}
if (progress.result.code == resultCode.bladeQueueFull)
{
if (allowPartialAllocation)
{
continue;
}
throw new Exception("Not enough blades available to accomodate new VMs");
}
if (progress.result.code != resultCode.success)
{
if (allowPartialAllocation)
{
continue;
}
throw new Exception("Unexpected status while allocing: " + progress.result.code);
}
vmSpec vmSpec = director.svc.getVMByIP_withoutLocking(progress.bladeName);
bladeSpec vmServerSpec = director.svc.getBladeByIP_withoutLocking(vmSpec.parentBladeIP);
snapshotDetails snapshotInfo = director.svc.getCurrentSnapshotDetails(vmSpec.VMIP);
NASParams nasParams = director.svc.getNASParams();
hypervisor_vmware_FreeNAS newVM = utils.createHypForVM(vmSpec, vmServerSpec, snapshotInfo, nasParams);
ipUtils.ensurePortIsFree(vmSpec.kernelDebugPort);
newVM.setDisposalCallback(onDestruction);
if (!toRet.TryAdd(vmSpec.VMIP, newVM))
{
throw new Exception();
}
}
}
catch (Exception)
{
foreach (KeyValuePair <string, hypervisorWithSpec <hypSpec_vmware> > allocedBlades in toRet)
{
if (allocedBlades.Key != null)
{
try
{
director.svc.ReleaseBladeOrVM(allocedBlades.Key);
}
catch (Exception)
{
// ...
}
}
}
throw;
}
allocWatch.Stop();
Debug.WriteLine("Allocated all VMs, took " + allocWatch.Elapsed.ToString());
return(toRet);
}
}
