public void CopyTwice()
{
var info = util.GetUsernameAndPassword();
using (var s = new SSHConnection(info.Item1, info.Item2))
{
// Create a "complex" directory structure on the remote machine
s
.ExecuteCommand("mkdir -p /tmp/usergwatts/data")
.ExecuteCommand("mkdir -p /tmp/usergwatts/data/d1")
.ExecuteCommand("mkdir -p /tmp/usergwatts/data/d2")
.ExecuteCommand("echo hi1 > /tmp/usergwatts/data/f1")
.ExecuteCommand("echo hi2 > /tmp/usergwatts/data/d1/f2")
.ExecuteCommand("echo hi3 > /tmp/usergwatts/data/d2/f3")
.ExecuteCommand("echo hi4 > /tmp/usergwatts/data/d2/f4");
// Remove everything local
var d = new DirectoryInfo("./data");
if (d.Exists)
{
d.Delete(true);
d.Refresh();
}
d.Create();
// Do the copy
s.CopyRemoteDirectoryLocally("/tmp/usergwatts/data", d);
s.CopyRemoteDirectoryLocally("/tmp/usergwatts/data", d);
}
}
public void copyBadRemoteDirectoryLocal()
{
var info = util.GetUsernameAndPassword();
using (var s = new SSHConnection(info.Item1, info.Item2))
{
// Do the copy
var d = new DirectoryInfo("./databogus");
if (d.Exists)
{
d.Delete(true);
d.Refresh();
}
d.Create();
s.CopyRemoteDirectoryLocally("/tmp/usergwatts/databogusbogusbogusbogus", d);
}
}
public void FetchOneFileWithNoBackupOverNetwork()
{
// This is a real run (so it is slow), and it will fetch data over the network.
var d = new DirectoryInfo("FetchOneFileWithNoBackupOverNetwork");
if (d.Exists)
{
d.Delete(true);
}
d.Create();
var c = GenerateLocalConfigWithWorkingRemote(d);
Locator.DisableAllLocators(true);
var l = LocalMachine.GetLocation(c);
var r = l.GetDSInfo("user.gwatts:user.gwatts.301295.EVNT.1");
Assert.AreEqual(1, l.GetDS(r, rep => Console.WriteLine(rep), fslist => fslist.Take(1).ToArray(), null, 0).Length);
var dsdir = d.SubDir("user.gwatts.301295.EVNT.1");
Assert.IsTrue(dsdir.Exists);
Assert.AreEqual(1, dsdir.EnumerateFiles("*.root.*", SearchOption.AllDirectories).Count());
// Next, make sure that there are no files left over up on the remote machine.
var username = c["LinuxUserName"];
var machine = c["LinuxHost"];
using (var s = new SSHConnection(machine, username))
{
bool stillThere = false;
s.ExecuteCommand(string.Format("ls {0}", c["LinuxTempLocation"]),
outLine =>
{
Console.WriteLine("output of ls: " + outLine);
stillThere |= !outLine.Contains("cannot access");
}
);
Assert.IsFalse(stillThere);
}
}
/// <summary>
/// Create the connection, and setup the appropriate stuff for using AMI.
/// </summary>
/// <returns></returns>
private SSHConnection GetConnection()
{
if (_connection != null)
return _connection;
// Establish the connection
var sm = JobParser.GetSubmissionMachine();
var connection = new SSHConnection(sm.MachineName, sm.Username);
// Setup for using ami.
connection.Apply(() => DisplayStatus("Setting up ATLAS"))
.setupATLAS()
.Apply(() => DisplayStatus("Setting up pyAMI"))
.ExecuteCommand("lsetup pyami")
.Apply(() => DisplayStatus("Acquiring GRID credentials"))
.VomsProxyInit("atlas", failNow: () => Stopping);
// And the connection is now ready for use!
_connection = connection;
return _connection;
}
/// <summary>
/// Load up the job requested. Fail, obviously, if we can't.
/// </summary>
protected override void ProcessRecord()
{
// Setup for verbosity if we need it.
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
// Get the job
var job = JobParser.FindJob(JobName, JobVersion);
// Get the expected resulting dataset name. Since this will be a personal
// dataset, we need to get the GRID info.
string ds = job.ResultingDatasetName(DatasetName, _gridCredentials);
// See if there is already a job defined that will produce this
var pandaJob = (ds + "/").FindPandaJobWithTaskName();
if (pandaJob == null)
{
// Where are we going to be doing the submission on?
var sm = JobParser.GetSubmissionMachine();
// Get the remove environment configured if it needs to be
var firstJob = false;
if (_connection == null)
{
firstJob = true;
_connection = new SSHConnection(sm.MachineName, sm.Username);
_connection
.Apply(() => DisplayStatus("Setting up ATLAS"))
.setupATLAS(dumpOnly: WhatIf.IsPresent)
.Apply(() => DisplayStatus("Setting up Rucio"))
.setupRucio(_gridCredentials.Username, dumpOnly: WhatIf.IsPresent)
.Apply(() => DisplayStatus("Acquiring GRID credentials"))
.VomsProxyInit("atlas", failNow: () => Stopping, dumpOnly: WhatIf.IsPresent);
}
// Check to see if the original dataset exists. We will use the location known as Local for doing the
// setup, I suppose.
var files = _connection
.Apply(() => DisplayStatus("Checking dataset exists on the GRID"))
.FilelistFromGRID(DatasetName, failNow: () => Stopping, dumpOnly: WhatIf.IsPresent);
if (files.Length == 0 && !WhatIf.IsPresent)
{
throw new ArgumentException($"Dataset '{DatasetName}' has zero files - won't submit a job against it!");
}
// Submit the job
_connection
.SubmitJob(job, DatasetName, ds, DisplayStatus, failNow: () => Stopping, sameJobAsLastTime: !firstJob, dumpOnly: WhatIf.IsPresent);
// Try to find the job again if requested. The submission can take a very long time to show up in
// big panda, so skip unless requested.
if (WaitForPandaRegistration)
{
var pandJobResult = Policy
.Handle<InvalidOperationException>()
.WaitAndRetryForever(nthRetry => TimeSpan.FromMinutes(1),
(e, ts) => { WriteWarning($"Failed to find the submitted panda job on bigpanda: {e.Message}. Will wait one minute and try again."); })
.ExecuteAndCapture(() => FindPandaJobForDS(ds));
if (pandJobResult.Outcome != OutcomeType.Successful)
{
throw pandJobResult.FinalException;
}
pandaJob = pandJobResult.Result;
}
}
// Return a helper obj that contains the info about this job that can be used by other commands.
if (pandaJob != null)
{
var r = new AtlasPandaTaskID() { ID = pandaJob.jeditaskid, Name = pandaJob.taskname };
WriteObject(r);
}
} finally
{
Trace.Listeners.Remove(listener);
}
}
/// <summary>
/// Initi with context info
/// </summary>
/// <param name="prompt"></param>
/// <param name="sSHConnection"></param>
public SSHSubShellContext(string prompt, SSHConnection sSHConnection)
{
this._oldPrompt = prompt;
this._connection = sSHConnection;
}
/// <summary>
/// Process a single dataset and fetch the info about it.
/// </summary>
protected override void ProcessRecord()
{
var cHit = _resultsCache.Value[DatasetName] as PSGRIDDatasetInfo;
if (cHit != null)
{
WriteObject(cHit);
}
else
{
// Setup for verbosity if we need it.
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
// Where are we going to be doing query on - we need a machine.
var sm = JobParser.GetSubmissionMachine();
// Get the remove environment configured if it needs to be
if (_connection == null)
{
_connection = new SSHConnection(sm.MachineName, sm.Username);
_connection
.Apply(() => DisplayStatus("Setting up ATLAS"))
.setupATLAS()
.Apply(() => DisplayStatus("Setting up Rucio"))
.setupRucio(_gridCredentials.Username)
.Apply(() => DisplayStatus("Acquiring GRID credentials"))
.VomsProxyInit("atlas", failNow: () => Stopping);
}
// Great - get the info on this dataset.
var fileInfo = _connection
.Apply(() => DisplayStatus($"Checking for info on {DatasetName}."))
.FileInfoFromGRID(DatasetName, failNow: () => Stopping);
// Next, build the resulting thingy.
var r = new PSGRIDDatasetInfo()
{
DatasetName = DatasetName,
nFiles = fileInfo.Count,
TotalSizeMB = (int)fileInfo.Sum(fi => fi.size),
FileInfo = fileInfo.ToArray()
};
_resultsCache.Value[DatasetName] = r;
WriteObject(r);
}
finally
{
Trace.Listeners.Remove(listener);
}
}
}
public void ReadBackWithPrompt()
{
var info = util.GetUsernameAndPassword();
using (var s = new SSHConnection(info.Item1, info.Item2))
{
Console.WriteLine("Before we do anything here is the environment:");
s.ExecuteCommand("set", l => Console.WriteLine(" set: " + l));
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();
bool valueSet = false;
bool sawPrompt = false;
s.ExecuteCommand("prompt=\"what is up: \"");
s.ExecuteCommandWithInput("read -p \"$prompt\" bogusvalue", new Dictionary<string, string>() { { "up:", "\nthis freak me out is a test" } }, l =>
{
sawPrompt = sawPrompt || l.Contains("what is up");
Console.WriteLine("==> " + l);
})
.ExecuteCommand("set", l => Console.WriteLine(" set: " + l))
.ExecuteCommand("echo bogusvalue $bogusvalue", l =>
{
valueSet = valueSet || l.Contains("this");
Console.WriteLine("--> " + l);
});
// THis guy isn't working yet because we don't seem to read in any input.
Assert.Inconclusive();
Assert.IsTrue(sawPrompt);
Assert.IsTrue(valueSet);
}
}
public void UnknownUsername()
{
using (var s = new SSHConnection("junk.washington.edu", "bogus-man"))
{ }
}
public void PWDOutputAsExpected()
{
var info = util.GetUsernameAndPassword();
using (var s = new SSHConnection(info.Item1, info.Item2))
{
int count = 0;
s.ExecuteCommand("pwd", l =>
{
Assert.IsFalse(l.Contains("\r"));
Assert.IsFalse(l.Contains("\n"));
Console.WriteLine("--> " + l);
count++;
});
Assert.AreEqual(1, count);
}
}
public void ListDirectoryWithNoOutput()
{
var info = util.GetUsernameAndPassword();
using (var s = new SSHConnection(info.Item1, info.Item2))
{
s.ExecuteCommand("ls -a | cat");
}
}
public void ListDirectory()
{
var info = util.GetUsernameAndPassword();
using (var s = new SSHConnection(info.Item1, info.Item2))
{
bool foundFile = false;
s.ExecuteCommand("ls -a | cat", l => { if (l.Contains(".bash_profile")) { foundFile = true; } Console.WriteLine(l); });
Assert.IsTrue(foundFile);
}
}
public void KnownUserAndCTor()
{
var info = util.GetUsernameAndPassword();
using (var s = new SSHConnection(info.Item1, info.Item2)) { }
}