private void onDestruction(hypSpec_vmware obj)
{
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
director.svc.ReleaseBladeOrVM(obj.kernelDebugIPOrHostname);
}
}
protected void Page_Load(object sender, EventArgs e)
{
if (!Request.QueryString.HasKeys())
{
return;
}
string ip = Request.QueryString["bladeip"];
string serverURL = status.getCurrentServerURL(this);
if (serverURL == null)
{
Response.Write("you're not logged in");
}
else
{
using (BladeDirectorServices services = new BladeDirectorServices(serverURL))
{
var spec = services.svc.getBladeByIP_withoutLocking(ip);
using (
hypervisor_iLo_HTTP theIlo = new hypervisor_iLo_HTTP(spec.iLOIP, spec.iLoUsername,
spec.iLoPassword))
{
theIlo.logoutOnDisposal = false;
theIlo.connect();
Response.Redirect(theIlo.makeHPLOLink());
}
}
}
}
protected void cmdAddNode_Click(object sender, EventArgs e)
{
using (BladeDirectorServices services = new BladeDirectorServices(getCurrentServerURL(this)))
{
services.svc.addNode(txtNewNodeIP.Text, txtNewISCSI.Text, txtNewIloIP.Text, UInt16.Parse(txtNewPort.Text), "TODO.the.key.here", "name of " + txtNewNodeIP.Text);
}
Response.Redirect(Request.RawUrl);
}
private void onDestruction(hypSpec_iLo obj)
{
// Notify the director that this blade is no longer in use.
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
director.svc.ReleaseBladeOrVM(obj.kernelDebugIPOrHostname);
}
}
public void setBIOSConfig(string newBiosXML)
{
hypSpec_iLo spec = getConnectionSpec();
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
resultAndWaitToken res = director.svc.rebootAndStartDeployingBIOSToBlade(spec.kernelDebugIPOrHostname, newBiosXML);
director.waitForSuccess(res, TimeSpan.FromMinutes(3));
}
}
public hypervisorCollection <hypSpec_iLo> requestAsManyHypervisorsAsPossible(string snapshotName)
{
initialiseIfNeeded();
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
int nodeCount = director.svc.getAllBladeIP().Length;
hypervisorCollection <hypSpec_iLo> toRet = new hypervisorCollection <hypSpec_iLo>();
DateTime deadline = DateTime.Now + TimeSpan.FromMinutes(60);
try
{
for (int i = 0; i < nodeCount; i++)
{
resultAndBladeName res = director.svc.RequestAnySingleNode();
resultAndBladeName progress = director.waitForSuccess(res, deadline - DateTime.Now);
bladeSpec bladeConfig = director.svc.getBladeByIP_withoutLocking(progress.bladeName);
resultAndWaitToken snapRes = director.svc.selectSnapshotForBladeOrVM(progress.bladeName, snapshotName);
director.waitForSuccess(snapRes, TimeSpan.FromMinutes(3));
snapshotDetails snapshot = director.svc.getCurrentSnapshotDetails(progress.bladeName);
userDesc cred = bladeConfig.credentials.First();
NASParams nas = director.svc.getNASParams();
hypSpec_iLo spec = new hypSpec_iLo(
bladeConfig.bladeIP, cred.username, cred.password,
bladeConfig.iLOIP, bladeConfig.iLoUsername, bladeConfig.iLoPassword,
nas.IP, nas.username, nas.password,
snapshot.friendlyName, snapshot.path,
bladeConfig.kernelDebugPort, bladeConfig.kernelDebugKey
);
ipUtils.ensurePortIsFree(bladeConfig.kernelDebugPort);
bladeDirectedHypervisor_iLo newHyp = new bladeDirectedHypervisor_iLo(spec);
newHyp.setDisposalCallback(onDestruction);
if (!toRet.TryAdd(bladeConfig.bladeIP, newHyp))
{
throw new Exception();
}
}
return(toRet);
}
catch (Exception)
{
toRet.Dispose();
throw;
}
}
}
private void keepaliveThreadMain()
{
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
while (true)
{
Thread.Sleep(TimeSpan.FromSeconds(3));
try
{
director.svc.keepAlive();
}
catch (TimeoutException)
{
}
}
}
}
private void updateNow()
{
lock (this)
{
using (BladeDirectorServices svc = new BladeDirectorServices(_baseURL))
{
foreach (ownedBlade blade in _blades)
{
GetBladeStatusResult bladeStatus = svc.svc.GetBladeStatus(blade.bladeIPAddress);
if (bladeStatus == GetBladeStatusResult.releasePending)
{
blade.isReleaseRequested = true;
}
else
{
blade.isReleaseRequested = false;
}
}
}
}
}
private void initialiseIfNeeded()
{
if (!doesCallStackHasTrait("requiresBladeDirector"))
{
Assert.Fail("This test uses the blade director; please decorate it with [TestCategory(\"requiresBladeDirector\")] for easier maintenence");
}
if (keepaliveThread == null)
{
lock (keepaliveThreadLock)
{
if (keepaliveThread == null)
{
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
resultAndWaitToken waitToken = director.svc.logIn();
// If there's a login already going for this machine, we will just use that one.
if (waitToken.result.code == resultCode.alreadyInProgress)
{
waitToken.result.code = resultCode.pending;
}
director.waitForSuccess(waitToken, TimeSpan.FromMinutes(10));
}
keepaliveThread = new Thread(keepaliveThreadMain);
keepaliveThread.Name = "Blade director keepalive thread";
keepaliveThread.Start();
}
}
}
if (!isConnected)
{
lock (_connectionLock)
{
isConnected = true;
}
}
}
private void doLoggedInData()
{
// Populate the main page data.
lblServerURL.Text = getCurrentServerURL(this);
lblClientIP.Text = Request.UserHostAddress;
TableRow headerRow = new TableRow();
headerRow.Cells.Add(new TableHeaderCell()
{
Text = ""
});
headerRow.Cells.Add(new TableHeaderCell()
{
Text = "State"
});
headerRow.Cells.Add(new TableHeaderCell()
{
Text = "Blade IP"
});
headerRow.Cells.Add(new TableHeaderCell()
{
Text = "Time since last keepalive"
});
headerRow.Cells.Add(new TableHeaderCell()
{
Text = "Currently-selected snapshot"
});
headerRow.Cells.Add(new TableHeaderCell()
{
Text = "Current owner"
});
headerRow.Cells.Add(new TableHeaderCell()
{
Text = "Next owner"
});
headerRow.Cells.Add(new TableHeaderCell()
{
Text = "Links"
});
tblBladeStatus.Rows.Add(headerRow);
using (BladeDirectorServices services = new BladeDirectorServices(getCurrentServerURL(this)))
{
lblServerBaseWebURL.Text = services.svc.getWebSvcURL();
string[] allBladeIPs = services.svc.getAllBladeIP();
foreach (string bladeIP in allBladeIPs)
{
// Query the server for information about this blade
bladeSpec bladeInfo = services.svc.getBladeByIP_withoutLocking(bladeIP);
// Assemble the always-visible status row
TableRow newRow = new TableRow();
newRow.Cells.Add(makeTableCell(new ImageButton()
{
ImageUrl = "images/collapsed.png",
AlternateText = "Details",
OnClientClick = "javascript:toggleDetail($(this), " + bladeInfo.bladeID + "); return false;"
}));
newRow.Cells.Add(new TableCell()
{
Text = bladeInfo.state.ToString()
});
newRow.Cells.Add(new TableCell()
{
Text = bladeInfo.bladeIP
});
if (bladeInfo.lastKeepAlive == DateTime.MinValue)
{
newRow.Cells.Add(new TableCell()
{
Text = "(none)"
});
}
else
{
string cssClass = "";
if (DateTime.Now - bladeInfo.lastKeepAlive > services.svc.getKeepAliveTimeout())
{
cssClass = "timedout";
}
TableCell cell = new TableCell
{
Text = formatDateTimeForWeb((DateTime.Now - bladeInfo.lastKeepAlive)),
CssClass = cssClass
};
newRow.Cells.Add(cell);
}
newRow.Cells.Add(new TableCell()
{
Text = bladeInfo.currentSnapshot
});
newRow.Cells.Add(new TableCell()
{
Text = bladeInfo.currentOwner ?? "none"
});
newRow.Cells.Add(new TableCell()
{
Text = bladeInfo.nextOwner ?? "none"
});
string iloURL = String.Format("iloConsole.aspx?bladeIP={0}", bladeInfo.bladeIP);
HyperLink link = new HyperLink()
{
NavigateUrl = iloURL, Text = "iLo"
};
TableCell iloURLtableCell = new TableCell();
iloURLtableCell.Controls.Add(link);
newRow.Cells.Add(iloURLtableCell);
tblBladeStatus.Rows.Add(newRow);
// Then populate the invisible-until-expanded details row.
tblBladeStatus.Rows.Add(makeDetailRow(services, bladeInfo));
}
// Finally, populate any log events.
logEntry[] logEvents = services.svc.getLogEvents(100);
int logID = 0;
foreach (logEntry logEvent in logEvents)
{
// Make the expandable row, with a little bit of info
TableRow tumbRow = new TableRow();
tumbRow.Cells.Add(makeTableCell(new ImageButton()
{
ImageUrl = "images/collapsed.png",
AlternateText = "Details",
OnClientClick = "javascript:toggleDetail($(this), null); return false;"
}));
tumbRow.Cells.Add(new TableCell()
{
Text = logEvent.timestamp.ToString()
});
tumbRow.Cells.Add(new TableCell()
{
Text = logEvent.threadName ?? "(unknown)"
});
string thumbtext = logEvent.msg.Split('\n')[0];
if (thumbtext.Length > 50)
{
thumbtext = thumbtext.Substring(0, 50) + "...";
}
tumbRow.Cells.Add(new TableCell()
{
Text = thumbtext
});
tblLogTable.Rows.Add(tumbRow);
// and then the full detail row.
TableRow detailRow = new TableRow();
detailRow.Cells.Add(new TableCell()
{
Text = logEvent.msg, ColumnSpan = 4
});
detailRow.Style.Add("display", "none");
tblLogTable.Rows.Add(detailRow);
logID++;
}
}
}
private TableRow makeDetailRow(BladeDirectorServices svc, bladeSpec bladeInfo)
{
Table detailTable = new Table();
TableRow miscTR = new TableRow();
miscTR.Cells.Add(makeTableCell(
new Label()
{
Text = "Blade DB ID: "
},
new Label()
{
Text = bladeInfo.bladeID.ToString() + "<br/>", CssClass = "fixedSize"
},
new Label()
{
Text = "ISCSI IP: "
},
new Label()
{
Text = bladeInfo.iscsiIP.ToString() + "<br/>", CssClass = "fixedSize"
},
new Label()
{
Text = "Kernel debug port: "
},
new Label()
{
Text = bladeInfo.kernelDebugPort.ToString() + "<br/>", CssClass = "fixedSize"
},
new Label()
{
Text = "Is currently having BIOS config deployed: "
},
new Label()
{
Text = bladeInfo.currentlyHavingBIOSDeployed.ToString() + "<br/>", CssClass = "fixedSize"
},
new Label()
{
Text = "Is currently acting as VM server: "
},
new Label()
{
Text = bladeInfo.currentlyBeingAVMServer.ToString() + "<br/>", CssClass = "fixedSize"
}
));
detailTable.Rows.Add(miscTR);
// And add rows for any VMs.
vmSpec[] VMs = svc.svc.getVMByVMServerIP_nolocking(bladeInfo.bladeIP);
if (VMs.Length > 0)
{
TableRow VMHeaderRow = new TableRow();
VMHeaderRow.Cells.Add(new TableHeaderCell()
{
Text = "VM name"
});
VMHeaderRow.Cells.Add(new TableHeaderCell()
{
Text = "Time since last keepalive"
});
VMHeaderRow.Cells.Add(new TableHeaderCell()
{
Text = "VM IP"
});
VMHeaderRow.Cells.Add(new TableHeaderCell()
{
Text = "Current owner"
});
VMHeaderRow.Cells.Add(new TableHeaderCell()
{
Text = "Kernel debug info"
});
detailTable.Rows.Add(VMHeaderRow);
}
foreach (vmSpec vmInfo in VMs)
{
TableRow thisVMRow = new TableRow();
thisVMRow.Cells.Add(new TableCell()
{
Text = vmInfo.friendlyName
});
thisVMRow.Cells.Add(new TableCell()
{
Text = formatDateTimeForWeb((DateTime.Now - vmInfo.lastKeepAlive))
});
thisVMRow.Cells.Add(new TableCell()
{
Text = vmInfo.VMIP
});
thisVMRow.Cells.Add(new TableCell()
{
Text = vmInfo.currentOwner
});
string dbgStr = String.Format("Port {0} key \"{1}\"", vmInfo.kernelDebugPort, vmInfo.kernelDebugKey);
thisVMRow.Cells.Add(new TableCell()
{
Text = dbgStr
});
//thisVMRow.Cells.Add(new TableCell() { Text = vmInfo.currentSnapshot });
detailTable.Rows.Add(thisVMRow);
}
TableHeaderRow toRet = new TableHeaderRow();
TableCell paddingCell = new TableCell {
CssClass = "invisible"
};
toRet.Cells.Add(paddingCell);
TableCell tc = new TableCell();
tc.ColumnSpan = 7;
tc.Controls.Add(detailTable);
toRet.Cells.Add(tc);
toRet.Style.Add("display", "none");
return(toRet);
}
public hypervisorCollection <hypSpec_vmware> requestVMs(VMSpec[] specs, bool allowPartialAllocation = false)
{
initialiseIfNeeded();
Stopwatch allocWatch = new Stopwatch();
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
DateTime deadline = DateTime.Now + TimeSpan.FromMinutes(30);
resultAndBladeName[] results = new resultAndBladeName[specs.Length];
for (int n = 0; n < specs.Length; n++)
{
results[n] = director.svc.RequestAnySingleVM(specs[n].hw, specs[n].sw);
if (results[n].result.code == resultCode.bladeQueueFull)
{
if (allowPartialAllocation)
{
results[n] = null;
break;
}
}
if (results[n].result.code != resultCode.success &&
results[n].result.code != resultCode.pending)
{
throw new bladeAllocationException(results[n].result.code);
}
}
hypervisorCollection <hypSpec_vmware> toRet = new hypervisorCollection <hypSpec_vmware>();
try
{
foreach (resultAndBladeName res in results)
{
if (res == null)
{
continue;
}
resultAndBladeName progress = director.waitForSuccessWithoutThrowing(res, deadline - DateTime.Now);
if (progress == null)
{
throw new TimeoutException();
}
if (progress.result.code == resultCode.bladeQueueFull)
{
if (allowPartialAllocation)
{
continue;
}
throw new Exception("Not enough blades available to accomodate new VMs");
}
if (progress.result.code != resultCode.success)
{
if (allowPartialAllocation)
{
continue;
}
throw new Exception("Unexpected status while allocing: " + progress.result.code);
}
vmSpec vmSpec = director.svc.getVMByIP_withoutLocking(progress.bladeName);
bladeSpec vmServerSpec = director.svc.getBladeByIP_withoutLocking(vmSpec.parentBladeIP);
snapshotDetails snapshotInfo = director.svc.getCurrentSnapshotDetails(vmSpec.VMIP);
NASParams nasParams = director.svc.getNASParams();
hypervisor_vmware_FreeNAS newVM = utils.createHypForVM(vmSpec, vmServerSpec, snapshotInfo, nasParams);
ipUtils.ensurePortIsFree(vmSpec.kernelDebugPort);
newVM.setDisposalCallback(onDestruction);
if (!toRet.TryAdd(vmSpec.VMIP, newVM))
{
throw new Exception();
}
}
}
catch (Exception)
{
foreach (KeyValuePair <string, hypervisorWithSpec <hypSpec_vmware> > allocedBlades in toRet)
{
if (allocedBlades.Key != null)
{
try
{
director.svc.ReleaseBladeOrVM(allocedBlades.Key);
}
catch (Exception)
{
// ...
}
}
}
throw;
}
allocWatch.Stop();
Debug.WriteLine("Allocated all VMs, took " + allocWatch.Elapsed.ToString());
return(toRet);
}
}
public bladeDirectedHypervisor_iLo createSingleHypervisor(string snapshotName)
{
initialiseIfNeeded();
// We request a blade from the blade director, and use them for all our tests, blocking if none are available.
using (BladeDirectorServices director = new BladeDirectorServices(machinePools.bladeDirectorURL))
{
// Request a node. If all queues are full, then wait and retry until we get one.
resultAndBladeName allocatedBladeResult;
while (true)
{
allocatedBladeResult = director.svc.RequestAnySingleNode();
if (allocatedBladeResult.result.code == resultCode.success ||
allocatedBladeResult.result.code == resultCode.pending)
{
break;
}
if (allocatedBladeResult.result.code == resultCode.bladeQueueFull)
{
Debug.WriteLine("All blades are fully queued, waiting until one is spare");
Thread.Sleep(TimeSpan.FromSeconds(5));
continue;
}
throw new Exception("Blade director returned unexpected return status '" + allocatedBladeResult.result + "'");
}
// Now, wait until our blade is available.
try
{
bool isMine = director.svc.isBladeMine(allocatedBladeResult.bladeName);
while (!isMine)
{
Debug.WriteLine("Blade " + allocatedBladeResult.bladeName + " not released yet, awaiting release..");
Thread.Sleep(TimeSpan.FromSeconds(5));
isMine = director.svc.isBladeMine(allocatedBladeResult.bladeName);
}
// Great, now we have ownership of the blade, so we can use it safely.
//bladeSpec bladeConfig = director.svc.getConfigurationOfBlade(allocatedBladeResult.bladeName);
resultAndWaitToken res = director.svc.selectSnapshotForBladeOrVM(allocatedBladeResult.bladeName, snapshotName);
director.waitForSuccess(res, TimeSpan.FromMinutes(5));
bladeSpec bladeConfig = director.svc.getBladeByIP_withoutLocking(allocatedBladeResult.bladeName);
snapshotDetails currentShot = director.svc.getCurrentSnapshotDetails(allocatedBladeResult.bladeName);
userDesc cred = bladeConfig.credentials.First();
NASParams nas = director.svc.getNASParams();
hypSpec_iLo spec = new hypSpec_iLo(
bladeConfig.bladeIP, cred.username, cred.password,
bladeConfig.iLOIP, bladeConfig.iLoUsername, bladeConfig.iLoPassword,
nas.IP, nas.username, nas.password,
currentShot.friendlyName, currentShot.path,
bladeConfig.kernelDebugPort, bladeConfig.kernelDebugKey
);
ipUtils.ensurePortIsFree(bladeConfig.kernelDebugPort);
bladeDirectedHypervisor_iLo toRet = new bladeDirectedHypervisor_iLo(spec);
toRet.setDisposalCallback(onDestruction);
return(toRet);
}
catch (Exception)
{
director.svc.ReleaseBladeOrVM(allocatedBladeResult.bladeName);
throw;
}
}
}
