public override void mkdir(string newDir, cancellableDateTime deadline)
{
// TODO: timeouts
NamePasswordAuthentication Auth = new NamePasswordAuthentication
{
Username = _spec.kernelVMUsername,
Password = _spec.kernelVMPassword,
InteractiveSession = true
};
VimClientImpl _vClient = conn.getConnection();
VirtualMachine _underlyingVM = conn.getMachine();
GuestOperationsManager gom = (GuestOperationsManager)_vClient.GetView(_vClient.ServiceContent.GuestOperationsManager, null);
GuestAuthManager guestAuthManager = _vClient.GetView(gom.AuthManager, null) as GuestAuthManager;
guestAuthManager.ValidateCredentialsInGuest(_underlyingVM.MoRef, Auth);
GuestFileManager GFM = _vClient.GetView(gom.FileManager, null) as GuestFileManager;
GFM.MakeDirectoryInGuest(_underlyingVM.MoRef, Auth, newDir, true);
}
private T doReqForJSON <T>(string url, string method, HttpStatusCode expectedCode, string payload = null, cancellableDateTime deadline = null)
{
if (deadline == null)
{
deadline = new cancellableDateTime(TimeSpan.FromMinutes(4));
}
while (true)
{
resp r = doReq(url, method, expectedCode, payload, deadline);
if (r.text.Contains("The request has timed out"))
{
deadline.doCancellableSleep(TimeSpan.FromSeconds(3));
}
else
{
return(JsonConvert.DeserializeObject <T>(r.text));
}
}
}
public override executionResult startExecutable(string toExecute, string args, string workingDir = null, cancellableDateTime deadline = null)
{
if (workingDir == null)
{
workingDir = "C:\\";
}
if (deadline == null)
{
deadline = new cancellableDateTime();
}
ProcessStartInfo ps = new ProcessStartInfo(toExecute, args);
ps.UseShellExecute = false;
ps.RedirectStandardError = true;
ps.RedirectStandardOutput = true;
ps.WorkingDirectory = workingDir;
using (Process process = Process.Start(ps))
{
TimeSpan timeout = deadline.getRemainingTimespan();
int timeoutMS;
if (timeout.TotalMilliseconds > int.MaxValue)
{
timeoutMS = -1;
}
else
{
timeoutMS = (int)timeout.TotalMilliseconds;
}
if (!process.WaitForExit(timeoutMS))
{
throw new TimeoutException();
}
return(new executionResult(process));
}
}
public void waitForPingability(bool waitForState, cancellableDateTime deadline = null)
{
if (deadline == null)
{
deadline = new cancellableDateTime();
}
// Wait for the box to go down/come up.
Debug.Print("Waiting for box " + getBaseConnectionSpec().kernelDebugIPOrHostname + " to " + (waitForState ? "come up" : "go down"));
while (true)
{
deadline.throwIfTimedOutOrCancelled();
if (waitForState)
{
Icmp pinger = new Icmp(Dns.GetHostAddresses(getBaseConnectionSpec().kernelDebugIPOrHostname).First());
if (pinger.Ping(TimeSpan.FromMilliseconds(500)))
{
Debug.Print(".. Box " + getBaseConnectionSpec().kernelDebugIPOrHostname + " pingable, giving it a few more seconds..");
deadline.doCancellableSleep(TimeSpan.FromSeconds(10));
Debug.Print(".. Box " + getBaseConnectionSpec().kernelDebugIPOrHostname + " assumed to be up now.");
break;
}
}
else
{
if (getPowerStatus() == false)
{
break;
}
}
deadline.doCancellableSleep(TimeSpan.FromSeconds(5));
}
Debug.Print(".. wait complete for box " + getBaseConnectionSpec().kernelDebugIPOrHostname);
}
public override void WaitForStatus(bool isPowerOn, cancellableDateTime deadline)
{
if (isPowerOn)
{
doWithRetryOnSomeExceptions(() =>
{
_executor.testConnectivity();
return(0);
}, deadline);
}
else
{
while (true)
{
if (getPowerStatus() == false)
{
break;
}
deadline.doCancellableSleep(TimeSpan.FromSeconds(5), "Failed to turn off machine via iLo");
}
}
}
public override void powerOff(cancellableDateTime deadline = null)
{
if (deadline == null)
{
deadline = new cancellableDateTime();
}
// Sometimes I am seeing 'the attempted operation cannot be performed in the current state (Powered on)' here,
// particularly under load, hence the retries.
while (VClient.getMachine().Runtime.PowerState != VirtualMachinePowerState.poweredOff)
{
doWithRetryOnSomeExceptions(() =>
{
VirtualMachine vm = VClient.getMachine();
vm.UpdateViewData();
if (vm.Runtime.PowerState == VirtualMachinePowerState.poweredOff)
{
return;
}
vm.PowerOffVM();
}, deadline, TimeSpan.FromSeconds(5));
}
}
private resp doReq(string url, string method, HttpStatusCode expectedCode, string payload = null, cancellableDateTime deadline = null, TimeSpan timeout = default(TimeSpan))
{
if (deadline == null)
{
deadline = new cancellableDateTime(TimeSpan.FromMinutes(4));
}
while (true)
{
resp toRet = _doReq(url, method, expectedCode, payload, timeout);
if (toRet != null)
{
return(toRet);
}
if (!deadline.stillOK)
{
throw new nasAccessException();
}
deadline.doCancellableSleep(TimeSpan.FromSeconds(10));
}
}
public override void WaitForStatus(bool isPowerOn, cancellableDateTime deadline)
{
if (isPowerOn)
{
doWithRetryOnSomeExceptions(() =>
{
executor.testConnectivity();
return(0);
}, deadline);
}
else
{
while (true)
{
if (getPowerStatus() == false)
{
break;
}
deadline.throwIfTimedOutOrCancelled();
deadline.doCancellableSleep(TimeSpan.FromSeconds(1));
}
}
}
public override void powerOn(cancellableDateTime deadline)
{
// Sometimes I am seeing 'the attempted operation cannot be performed in the current state (Powered on)' here,
// particularly under load, hence the retries.
doWithRetryOnSomeExceptions(() =>
{
VirtualMachine VM = VClient.getMachine();
if (VM.Runtime.PowerState == VirtualMachinePowerState.poweredOn)
{
return;
}
VM.PowerOnVM(VM.Runtime.Host);
}, deadline, TimeSpan.FromSeconds(5));
// Wait for it to be ready
if (_executionMethod == clientExecutionMethod.vmwaretools)
{
while (true)
{
VirtualMachine VM = VClient.getMachine();
if (VM.Guest.ToolsRunningStatus == VirtualMachineToolsRunningStatus.guestToolsRunning.ToString())
{
break;
}
deadline.doCancellableSleep(TimeSpan.FromSeconds(3));
}
}
// No, really, wait for it to be ready
doWithRetryOnSomeExceptions(() =>
{
executor.testConnectivity();
return("");
}, deadline, TimeSpan.FromSeconds(5));
}
public abstract executionResult startExecutable(string toExecute, string args, string workingdir = null, cancellableDateTime deadline = null);
public abstract string getFileFromGuest(string srcpath, cancellableDateTime deadline = null);
public abstract void copyToGuest(string dstpath, string srcpath, cancellableDateTime deadline = null);
public abstract void WaitForStatus(bool isPowerOn, cancellableDateTime deadline);
public abstract void powerOff(cancellableDateTime deadline);
public virtual executionResult startExecutable(string toExecute, string args, string workingDir = null, cancellableDateTime deadline = null)
{
if (deadline == null)
{
deadline = new cancellableDateTime(TimeSpan.FromMinutes(3));
}
IAsyncExecutionResult resultInProgress = null;
try
{
while (resultInProgress == null)
{
resultInProgress = startExecutableAsync(toExecute, args, workingDir);
// FIXME!!!
// This is causing an awful assembly load failure when it throws :/
// Disabling the timeout for now because I am waaay behind schedule already
// if (DateTime.Now > deadline)
// throw new hypervisorExecutionException();
deadline.doCancellableSleep(TimeSpan.FromSeconds(3));
}
while (true)
{
deadline.doCancellableSleep(TimeSpan.FromSeconds(3));
executionResult res = resultInProgress.getResultIfComplete();
if (res != null)
{
return(res);
}
}
}
finally
{
if (resultInProgress != null)
{
resultInProgress.Dispose();
}
}
}
public override void powerOff(cancellableDateTime deadline)
{
}
public override void mkdir(string newDir, cancellableDateTime deadline)
{
throw new NotImplementedException();
}
public static T doWithRetryOnSomeExceptions <T>(Func <SMBExecutor.triedNetworkCallRes <T> > thingtoDo,
cancellableDateTime deadline = null, TimeSpan retryDelay = default(TimeSpan))
{
if (retryDelay == default(TimeSpan))
{
retryDelay = TimeSpan.FromSeconds(1);
}
if (deadline == null)
{
deadline = new cancellableDateTime();
}
while (true)
{
SMBExecutor.triedNetworkCallRes <T> res = thingtoDo.Invoke();
if (!res.retryRequested)
{
if (res.error == null)
{
return(res.res);
}
if (res.error is VimException)
{
// This VimException is fatal
if (res.error.Message.Contains("are insufficient for the operation") ||
res.error.Message.Contains("Permission to perform this operation was denied."))
{
throw res.error;
}
// while others are not.
if (!deadline.stillOK)
{
throw res.error;
}
}
if (!(res.error is SocketException) &&
!(res.error is SoapException) &&
!(res.error is TimeoutException) &&
!(res.error is WebException) &&
!(res.error is hypervisorExecutionException) &&
!(res.error is hypervisorExecutionException_retryable) &&
!(res.error is SshOperationTimeoutException) &&
!(res.error is SshConnectionException) &&
!(res.error is UnauthorizedAccessException) &&
!(res.error is COMException))
{
// An exception of a type we don't anticipate, so throw
throw res.error;
}
// throw if the deadline has passed
if (!deadline.stillOK)
{
throw res.error;
}
}
// Otherwise, just retry.
deadline.doCancellableSleep(retryDelay);
}
}
public cancellableDateTime(TimeSpan newTimeout, cancellableDateTime newChildDeadline)
{
deadline = DateTime.Now + newTimeout;
childDeadline = newChildDeadline;
isCancelled = false;
}
public override executionResult startExecutable(string toExecute, string args, string workingdir = null, cancellableDateTime deadline = null)
{
if (deadline == null)
{
deadline = new cancellableDateTime();
}
return(executor.startExecutable(toExecute, args, workingdir, deadline));
}
public override void WaitForStatus(bool isPowerOn, cancellableDateTime deadline)
{
}
public override executionResult startExecutable(string toExecute, string args, string workingdir = null, cancellableDateTime deadline = null)
{
if (_executor == null)
{
throw new NotSupportedException();
}
executionResult toRet = _executor.startExecutable(toExecute, args, workingdir, deadline);
//Debug.WriteLine("Command '{0}' with args '{1}' returned {2} stdout '{3}' stderr '{4}'", toExecute, args, toRet.resultCode, toRet.stdout, toRet.stderr);
return(toRet);
}
public override string getFileFromGuest(string srcpath, cancellableDateTime deadline)
{
return(File.ReadAllText(srcpath));
}
public static void restoreSnapshot <T>(hypervisorWithSpec <T> hyp, FreeNASWithCaching nas, cancellableDateTime deadline)
{
hypSpec_withWindbgKernel _spec = hyp.getBaseConnectionSpec();
// Find the device snapshot, so we can get information about it needed to get the ISCSI volume
snapshotObjects shotObjects = getSnapshotObjectsFromNAS(nas, _spec.snapshotFullName);
// Here we power the server down, tell the iSCSI server to use the right image, and power it back up again.
hyp.powerOff();
// Now we can get started. We must remove the 'target to extent' mapping, then the extent. Then we can safely roll back
// the ZFS snapshot, and then re-add the extent and mapping. We use a finally block so that it is less likely we will
// leave the NAS object in an inconsistent state.
// Note that removing the target will also remove any 'target to extent' mapping, so we don't need to do that explicitly.
// FreeNAS will take care of it for us.
nas.deleteISCSIExtent(shotObjects.extent);
nas.waitUntilISCSIConfigFlushed();
try
{
// Roll back the snapshot. Use a retry, since FreeNAS is complaining the dataset is in use occasionally.
int retries = 100;
while (true)
{
try
{
nas.rollbackSnapshot(shotObjects.shotToRestore);
break;
}
catch (Exception)
{
if (retries-- == 0)
{
throw;
}
deadline.doCancellableSleep(TimeSpan.FromSeconds(6)); // 6 sec * 100 retries = ten minutes
}
}
}
finally
{
// Re-add the extent and target-to-extent mapping. Use the same target ID as the old target-to-extent used, and
// the new extent's ID.
iscsiExtent newExtent = nas.addISCSIExtent(shotObjects.extent);
nas.addISCSITargetToExtent(shotObjects.tgtToExtent.iscsi_target, newExtent);
nas.waitUntilISCSIConfigFlushed();
}
}
public abstract void mkdir(string newDir, cancellableDateTime deadline = null);
public T withRetryUntilSuccess <T>(Func <SMBExecutor.triedNetworkCallRes <T> > action, cancellableDateTime deadline)
{
while (true)
{
SMBExecutor.triedNetworkCallRes <T> res = action();
if (!res.retryRequested)
{
if (res.error == null)
{
return(res.res);
}
if (!(res.error is Win32Exception) &&
!(res.error is TimeoutException) &&
!(res.error is IOException) &&
!(res.error is VimException))
{
throw res.error;
}
if (!deadline.stillOK)
{
throw res.error;
}
}
deadline.doCancellableSleep(TimeSpan.FromSeconds(3));
}
}
public override executionResult startExecutable(string toExecute, string args, string workingDir = null, cancellableDateTime deadline = null)
{
if (workingDir != null)
{
throw new NotSupportedException();
}
// TODO: timeouts aren't supported here, they're just ignored.
try
{
using (SshClient client = new SshClient(inf))
{
client.Connect();
SshCommand returnVal = client.RunCommand(string.Format("{0} {1}", toExecute, args));
Debug.WriteLine("{0} : Command '{1}' args '{2}' retcode {3} stdout {4} stderr {5}", _hostIp, toExecute, args, returnVal.ExitStatus, returnVal.Result, returnVal.Error);
return(new executionResult(returnVal));
}
}
catch (SshException)
{
throw new hypervisorExecutionException();
}
}
public abstract void deleteFile(string toDelete, cancellableDateTime deadline);
public override void deleteFile(string toDelete, cancellableDateTime deadline)
{
throw new NotImplementedException();
}
public override void mkdir(string newDir, cancellableDateTime deadline)
{
Directory.CreateDirectory(newDir);
}
