/// <summary>
/// Setup!
/// </summary>
protected override void BeginProcessing()
{
// Setup for verbosity if we need it.
_listener = new PSListener(this);
Trace.Listeners.Add(_listener);
base.BeginProcessing();
}
/// <summary>
/// Get the list and write it out as objects.
/// </summary>
protected override void ProcessRecord()
{
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
var list = GRIDDatasetLocator.GetActiveLocations();
foreach (var l in list)
{
WriteObject(l.Name);
}
} finally
{
Trace.Listeners.Remove(listener);
}
}
/// <summary>
/// Fetch grid credentials for later use
/// </summary>
protected override void BeginProcessing()
{
// Setup for verbosity if we need it.
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
// Get the grid credentials
_gridCredentials = new CredentialSet("GRID").Load().FirstOrDefault();
if (_gridCredentials == null)
{
throw new ArgumentException("Please create a generic windows credential with the target 'GRID' with the username as the rucio grid username and the password to be used with voms proxy init");
}
}
finally
{
Trace.Listeners.Remove(listener);
}
}
/// <summary>
/// Fetch a particular dataset.
/// </summary>
protected override void ProcessRecord()
{
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
// First, deal with the file filter
Func<string[], string[]> filter = nFiles == 0 ? (Func<string[],string[]>) null : flist => flist.OrderBy(f => f).Take(nFiles).ToArray();
// Get the actual dataset name.
var dataset = DatasetName;
if (ParameterSetName == "job")
{
// Get the job, see if it is finished, and then get the output dataset.
var job = JobParser.FindJob(JobName, JobVersion);
if (job == null)
{
throw new ArgumentException($"Job {JobName} v{JobVersion} is not known to the system!");
}
// Get the resulting job name for this guy.
var pandaJobName = job.ResultingDatasetName(JobSourceDatasetName) + "/";
// Now, to get the output dataset, we need to fetch the job.
var pandaTask = pandaJobName.FindPandaJobWithTaskName(true);
if (pandaTask == null)
{
throw new ArgumentException($"No panda task found with name '{pandaJobName}' for job '{JobName}' v{JobVersion}.");
}
var containers = pandaTask.DatasetNamesOUT();
if (containers.Length > 1)
{
throw new ArgumentException($"There are more than one output container for the panda task {pandaTask.jeditaskid} - can't decide. Need code upgrade!! Thanks for the fish!");
}
dataset = containers.First();
}
// Now, how we pick up what we want will depend on if we are trying to download to a location.
Uri[] r = null;
if (!string.IsNullOrWhiteSpace(Location))
{
using (var holder = GRIDDatasetLocator.SetLocationFilter(loc => false))
{
r = GRIDDatasetLocator.FetchDatasetUrisAtLocation(Location, dataset, fname => DisplayStatus(fname), fileFilter: filter, failNow: () => Stopping, timeoutDownloadSecs: Timeout);
}
}
else
{
r = GRIDDatasetLocator.FetchDatasetUris(dataset, fname => DisplayStatus(fname), fileFilter: filter, failNow: () => Stopping, timeoutDownloadSecs: Timeout);
}
// Dump all the returned files out to whatever is next in the pipeline.
foreach (var ds in r)
{
WriteObject(ds);
}
}
finally
{
Trace.Listeners.Remove(listener);
}
}
/// <summary>
/// Load up the job requested. Fail, obviously, if we can't.
/// </summary>
protected override void ProcessRecord()
{
// Setup for verbosity if we need it.
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
// Get the job
var job = JobParser.FindJob(JobName, JobVersion);
// Get the expected resulting dataset name. Since this will be a personal
// dataset, we need to get the GRID info.
string ds = job.ResultingDatasetName(DatasetName, _gridCredentials);
// See if there is already a job defined that will produce this
var pandaJob = (ds + "/").FindPandaJobWithTaskName();
if (pandaJob == null)
{
// Where are we going to be doing the submission on?
var sm = JobParser.GetSubmissionMachine();
// Get the remove environment configured if it needs to be
var firstJob = false;
if (_connection == null)
{
firstJob = true;
_connection = new SSHConnection(sm.MachineName, sm.Username);
_connection
.Apply(() => DisplayStatus("Setting up ATLAS"))
.setupATLAS(dumpOnly: WhatIf.IsPresent)
.Apply(() => DisplayStatus("Setting up Rucio"))
.setupRucio(_gridCredentials.Username, dumpOnly: WhatIf.IsPresent)
.Apply(() => DisplayStatus("Acquiring GRID credentials"))
.VomsProxyInit("atlas", failNow: () => Stopping, dumpOnly: WhatIf.IsPresent);
}
// Check to see if the original dataset exists. We will use the location known as Local for doing the
// setup, I suppose.
var files = _connection
.Apply(() => DisplayStatus("Checking dataset exists on the GRID"))
.FilelistFromGRID(DatasetName, failNow: () => Stopping, dumpOnly: WhatIf.IsPresent);
if (files.Length == 0 && !WhatIf.IsPresent)
{
throw new ArgumentException($"Dataset '{DatasetName}' has zero files - won't submit a job against it!");
}
// Submit the job
_connection
.SubmitJob(job, DatasetName, ds, DisplayStatus, failNow: () => Stopping, sameJobAsLastTime: !firstJob, dumpOnly: WhatIf.IsPresent);
// Try to find the job again if requested. The submission can take a very long time to show up in
// big panda, so skip unless requested.
if (WaitForPandaRegistration)
{
var pandJobResult = Policy
.Handle<InvalidOperationException>()
.WaitAndRetryForever(nthRetry => TimeSpan.FromMinutes(1),
(e, ts) => { WriteWarning($"Failed to find the submitted panda job on bigpanda: {e.Message}. Will wait one minute and try again."); })
.ExecuteAndCapture(() => FindPandaJobForDS(ds));
if (pandJobResult.Outcome != OutcomeType.Successful)
{
throw pandJobResult.FinalException;
}
pandaJob = pandJobResult.Result;
}
}
// Return a helper obj that contains the info about this job that can be used by other commands.
if (pandaJob != null)
{
var r = new AtlasPandaTaskID() { ID = pandaJob.jeditaskid, Name = pandaJob.taskname };
WriteObject(r);
}
} finally
{
Trace.Listeners.Remove(listener);
}
}
/// <summary>
/// Init with the listener.
/// </summary>
/// <param name="listener"></param>
public PSListenerPause(PSListener listener)
{
_listener = listener;
_origianlState = listener.LogState;
_listener.LogState = false;
}
/// <summary>
/// Process a single dataset and fetch the info about it.
/// </summary>
protected override void ProcessRecord()
{
var cHit = _resultsCache.Value[DatasetName] as PSGRIDDatasetInfo;
if (cHit != null)
{
WriteObject(cHit);
}
else
{
// Setup for verbosity if we need it.
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
// Where are we going to be doing query on - we need a machine.
var sm = JobParser.GetSubmissionMachine();
// Get the remove environment configured if it needs to be
if (_connection == null)
{
_connection = new SSHConnection(sm.MachineName, sm.Username);
_connection
.Apply(() => DisplayStatus("Setting up ATLAS"))
.setupATLAS()
.Apply(() => DisplayStatus("Setting up Rucio"))
.setupRucio(_gridCredentials.Username)
.Apply(() => DisplayStatus("Acquiring GRID credentials"))
.VomsProxyInit("atlas", failNow: () => Stopping);
}
// Great - get the info on this dataset.
var fileInfo = _connection
.Apply(() => DisplayStatus($"Checking for info on {DatasetName}."))
.FileInfoFromGRID(DatasetName, failNow: () => Stopping);
// Next, build the resulting thingy.
var r = new PSGRIDDatasetInfo()
{
DatasetName = DatasetName,
nFiles = fileInfo.Count,
TotalSizeMB = (int)fileInfo.Sum(fi => fi.size),
FileInfo = fileInfo.ToArray()
};
_resultsCache.Value[DatasetName] = r;
WriteObject(r);
}
finally
{
Trace.Listeners.Remove(listener);
}
}
}