/// <summary> /// Setup! /// </summary> protected override void BeginProcessing() { // Setup for verbosity if we need it. _listener = new PSListener(this); Trace.Listeners.Add(_listener); base.BeginProcessing(); }
/// <summary> /// Get the list and write it out as objects. /// </summary> protected override void ProcessRecord() { var listener = new PSListener(this); Trace.Listeners.Add(listener); try { var list = GRIDDatasetLocator.GetActiveLocations(); foreach (var l in list) { WriteObject(l.Name); } } finally { Trace.Listeners.Remove(listener); } }
/// <summary> /// Fetch grid credentials for later use /// </summary> protected override void BeginProcessing() { // Setup for verbosity if we need it. var listener = new PSListener(this); Trace.Listeners.Add(listener); try { // Get the grid credentials _gridCredentials = new CredentialSet("GRID").Load().FirstOrDefault(); if (_gridCredentials == null) { throw new ArgumentException("Please create a generic windows credential with the target 'GRID' with the username as the rucio grid username and the password to be used with voms proxy init"); } } finally { Trace.Listeners.Remove(listener); } }
/// <summary> /// Fetch a particular dataset. /// </summary> protected override void ProcessRecord() { var listener = new PSListener(this); Trace.Listeners.Add(listener); try { // First, deal with the file filter Func<string[], string[]> filter = nFiles == 0 ? (Func<string[],string[]>) null : flist => flist.OrderBy(f => f).Take(nFiles).ToArray(); // Get the actual dataset name. var dataset = DatasetName; if (ParameterSetName == "job") { // Get the job, see if it is finished, and then get the output dataset. var job = JobParser.FindJob(JobName, JobVersion); if (job == null) { throw new ArgumentException($"Job {JobName} v{JobVersion} is not known to the system!"); } // Get the resulting job name for this guy. var pandaJobName = job.ResultingDatasetName(JobSourceDatasetName) + "/"; // Now, to get the output dataset, we need to fetch the job. var pandaTask = pandaJobName.FindPandaJobWithTaskName(true); if (pandaTask == null) { throw new ArgumentException($"No panda task found with name '{pandaJobName}' for job '{JobName}' v{JobVersion}."); } var containers = pandaTask.DatasetNamesOUT(); if (containers.Length > 1) { throw new ArgumentException($"There are more than one output container for the panda task {pandaTask.jeditaskid} - can't decide. Need code upgrade!! Thanks for the fish!"); } dataset = containers.First(); } // Now, how we pick up what we want will depend on if we are trying to download to a location. Uri[] r = null; if (!string.IsNullOrWhiteSpace(Location)) { using (var holder = GRIDDatasetLocator.SetLocationFilter(loc => false)) { r = GRIDDatasetLocator.FetchDatasetUrisAtLocation(Location, dataset, fname => DisplayStatus(fname), fileFilter: filter, failNow: () => Stopping, timeoutDownloadSecs: Timeout); } } else { r = GRIDDatasetLocator.FetchDatasetUris(dataset, fname => DisplayStatus(fname), fileFilter: filter, failNow: () => Stopping, timeoutDownloadSecs: Timeout); } // Dump all the returned files out to whatever is next in the pipeline. foreach (var ds in r) { WriteObject(ds); } } finally { Trace.Listeners.Remove(listener); } }
/// <summary> /// Load up the job requested. Fail, obviously, if we can't. /// </summary> protected override void ProcessRecord() { // Setup for verbosity if we need it. var listener = new PSListener(this); Trace.Listeners.Add(listener); try { // Get the job var job = JobParser.FindJob(JobName, JobVersion); // Get the expected resulting dataset name. Since this will be a personal // dataset, we need to get the GRID info. string ds = job.ResultingDatasetName(DatasetName, _gridCredentials); // See if there is already a job defined that will produce this var pandaJob = (ds + "/").FindPandaJobWithTaskName(); if (pandaJob == null) { // Where are we going to be doing the submission on? var sm = JobParser.GetSubmissionMachine(); // Get the remove environment configured if it needs to be var firstJob = false; if (_connection == null) { firstJob = true; _connection = new SSHConnection(sm.MachineName, sm.Username); _connection .Apply(() => DisplayStatus("Setting up ATLAS")) .setupATLAS(dumpOnly: WhatIf.IsPresent) .Apply(() => DisplayStatus("Setting up Rucio")) .setupRucio(_gridCredentials.Username, dumpOnly: WhatIf.IsPresent) .Apply(() => DisplayStatus("Acquiring GRID credentials")) .VomsProxyInit("atlas", failNow: () => Stopping, dumpOnly: WhatIf.IsPresent); } // Check to see if the original dataset exists. We will use the location known as Local for doing the // setup, I suppose. var files = _connection .Apply(() => DisplayStatus("Checking dataset exists on the GRID")) .FilelistFromGRID(DatasetName, failNow: () => Stopping, dumpOnly: WhatIf.IsPresent); if (files.Length == 0 && !WhatIf.IsPresent) { throw new ArgumentException($"Dataset '{DatasetName}' has zero files - won't submit a job against it!"); } // Submit the job _connection .SubmitJob(job, DatasetName, ds, DisplayStatus, failNow: () => Stopping, sameJobAsLastTime: !firstJob, dumpOnly: WhatIf.IsPresent); // Try to find the job again if requested. The submission can take a very long time to show up in // big panda, so skip unless requested. if (WaitForPandaRegistration) { var pandJobResult = Policy .Handle<InvalidOperationException>() .WaitAndRetryForever(nthRetry => TimeSpan.FromMinutes(1), (e, ts) => { WriteWarning($"Failed to find the submitted panda job on bigpanda: {e.Message}. Will wait one minute and try again."); }) .ExecuteAndCapture(() => FindPandaJobForDS(ds)); if (pandJobResult.Outcome != OutcomeType.Successful) { throw pandJobResult.FinalException; } pandaJob = pandJobResult.Result; } } // Return a helper obj that contains the info about this job that can be used by other commands. if (pandaJob != null) { var r = new AtlasPandaTaskID() { ID = pandaJob.jeditaskid, Name = pandaJob.taskname }; WriteObject(r); } } finally { Trace.Listeners.Remove(listener); } }
/// <summary> /// Init with the listener. /// </summary> /// <param name="listener"></param> public PSListenerPause(PSListener listener) { _listener = listener; _origianlState = listener.LogState; _listener.LogState = false; }
/// <summary> /// Process a single dataset and fetch the info about it. /// </summary> protected override void ProcessRecord() { var cHit = _resultsCache.Value[DatasetName] as PSGRIDDatasetInfo; if (cHit != null) { WriteObject(cHit); } else { // Setup for verbosity if we need it. var listener = new PSListener(this); Trace.Listeners.Add(listener); try { // Where are we going to be doing query on - we need a machine. var sm = JobParser.GetSubmissionMachine(); // Get the remove environment configured if it needs to be if (_connection == null) { _connection = new SSHConnection(sm.MachineName, sm.Username); _connection .Apply(() => DisplayStatus("Setting up ATLAS")) .setupATLAS() .Apply(() => DisplayStatus("Setting up Rucio")) .setupRucio(_gridCredentials.Username) .Apply(() => DisplayStatus("Acquiring GRID credentials")) .VomsProxyInit("atlas", failNow: () => Stopping); } // Great - get the info on this dataset. var fileInfo = _connection .Apply(() => DisplayStatus($"Checking for info on {DatasetName}.")) .FileInfoFromGRID(DatasetName, failNow: () => Stopping); // Next, build the resulting thingy. var r = new PSGRIDDatasetInfo() { DatasetName = DatasetName, nFiles = fileInfo.Count, TotalSizeMB = (int)fileInfo.Sum(fi => fi.size), FileInfo = fileInfo.ToArray() }; _resultsCache.Value[DatasetName] = r; WriteObject(r); } finally { Trace.Listeners.Remove(listener); } } }