예제 #1
0
        /// <summary>
        /// Setup!
        /// </summary>
        protected override void BeginProcessing()
        {
            // Setup for verbosity if we need it.
            _listener = new PSListener(this);
            Trace.Listeners.Add(_listener);

            base.BeginProcessing();
        }
예제 #2
0
 /// <summary>
 /// Get the list and write it out as objects.
 /// </summary>
 protected override void ProcessRecord()
 {
     var listener = new PSListener(this);
     Trace.Listeners.Add(listener);
     try
     {
         var list = GRIDDatasetLocator.GetActiveLocations();
         foreach (var l in list)
         {
             WriteObject(l.Name);
         }
     } finally
     {
         Trace.Listeners.Remove(listener);
     }
 }
예제 #3
0
 /// <summary>
 /// Fetch grid credentials for later use
 /// </summary>
 protected override void BeginProcessing()
 {
     // Setup for verbosity if we need it.
     var listener = new PSListener(this);
     Trace.Listeners.Add(listener);
     try
     {
         // Get the grid credentials
         _gridCredentials = new CredentialSet("GRID").Load().FirstOrDefault();
         if (_gridCredentials == null)
         {
             throw new ArgumentException("Please create a generic windows credential with the target 'GRID' with the username as the rucio grid username and the password to be used with voms proxy init");
         }
     }
     finally
     {
         Trace.Listeners.Remove(listener);
     }
 }
예제 #4
0
        /// <summary>
        /// Fetch a particular dataset.
        /// </summary>
        protected override void ProcessRecord()
        {
            var listener = new PSListener(this);
            Trace.Listeners.Add(listener);
            try
            {
                // First, deal with the file filter
                Func<string[], string[]> filter = nFiles == 0 ? (Func<string[],string[]>) null : flist => flist.OrderBy(f => f).Take(nFiles).ToArray();

                // Get the actual dataset name.
                var dataset = DatasetName;
                if (ParameterSetName == "job")
                {
                    // Get the job, see if it is finished, and then get the output dataset.
                    var job = JobParser.FindJob(JobName, JobVersion);
                    if (job == null)
                    {
                        throw new ArgumentException($"Job {JobName} v{JobVersion} is not known to the system!");
                    }

                    // Get the resulting job name for this guy.
                    var pandaJobName = job.ResultingDatasetName(JobSourceDatasetName) + "/";

                    // Now, to get the output dataset, we need to fetch the job.
                    var pandaTask = pandaJobName.FindPandaJobWithTaskName(true);
                    if (pandaTask == null)
                    {
                        throw new ArgumentException($"No panda task found with name '{pandaJobName}' for job '{JobName}' v{JobVersion}.");
                    }
                    var containers = pandaTask.DatasetNamesOUT();
                    if (containers.Length > 1)
                    {
                        throw new ArgumentException($"There are more than one output container for the panda task {pandaTask.jeditaskid} - can't decide. Need code upgrade!! Thanks for the fish!");
                    }
                    dataset = containers.First();
                }

                // Now, how we pick up what we want will depend on if we are trying to download to a location.
                Uri[] r = null;
                if (!string.IsNullOrWhiteSpace(Location))
                {
                    using (var holder = GRIDDatasetLocator.SetLocationFilter(loc => false))
                    {
                        r = GRIDDatasetLocator.FetchDatasetUrisAtLocation(Location, dataset, fname => DisplayStatus(fname), fileFilter: filter, failNow: () => Stopping, timeoutDownloadSecs: Timeout);
                    }
                }
                else
                {
                    r = GRIDDatasetLocator.FetchDatasetUris(dataset, fname => DisplayStatus(fname), fileFilter: filter, failNow: () => Stopping, timeoutDownloadSecs: Timeout);
                }

                // Dump all the returned files out to whatever is next in the pipeline.
                foreach (var ds in r)
                {
                    WriteObject(ds);
                }
            }
            finally
            {
                Trace.Listeners.Remove(listener);
            }
        }
예제 #5
0
        /// <summary>
        /// Load up the job requested. Fail, obviously, if we can't.
        /// </summary>
        protected override void ProcessRecord()
        {
            // Setup for verbosity if we need it.
            var listener = new PSListener(this);
            Trace.Listeners.Add(listener);
            try
            {
                // Get the job
                var job = JobParser.FindJob(JobName, JobVersion);

                // Get the expected resulting dataset name. Since this will be a personal
                // dataset, we need to get the GRID info.
                string ds = job.ResultingDatasetName(DatasetName, _gridCredentials);

                // See if there is already a job defined that will produce this
                var pandaJob = (ds + "/").FindPandaJobWithTaskName();

                if (pandaJob == null)
                {
                    // Where are we going to be doing the submission on?
                    var sm = JobParser.GetSubmissionMachine();

                    // Get the remove environment configured if it needs to be
                    var firstJob = false;
                    if (_connection == null)
                    {
                        firstJob = true;
                        _connection = new SSHConnection(sm.MachineName, sm.Username);
                        _connection
                            .Apply(() => DisplayStatus("Setting up ATLAS"))
                            .setupATLAS(dumpOnly: WhatIf.IsPresent)
                            .Apply(() => DisplayStatus("Setting up Rucio"))
                            .setupRucio(_gridCredentials.Username, dumpOnly: WhatIf.IsPresent)
                            .Apply(() => DisplayStatus("Acquiring GRID credentials"))
                            .VomsProxyInit("atlas", failNow: () => Stopping, dumpOnly: WhatIf.IsPresent);
                    }

                    // Check to see if the original dataset exists. We will use the location known as Local for doing the
                    // setup, I suppose.
                    var files = _connection
                        .Apply(() => DisplayStatus("Checking dataset exists on the GRID"))
                        .FilelistFromGRID(DatasetName, failNow: () => Stopping, dumpOnly: WhatIf.IsPresent);
                    if (files.Length == 0 && !WhatIf.IsPresent)
                    {
                        throw new ArgumentException($"Dataset '{DatasetName}' has zero files - won't submit a job against it!");
                    }

                    // Submit the job
                    _connection
                        .SubmitJob(job, DatasetName, ds, DisplayStatus, failNow: () => Stopping, sameJobAsLastTime: !firstJob, dumpOnly: WhatIf.IsPresent);

                    // Try to find the job again if requested. The submission can take a very long time to show up in
                    // big panda, so skip unless requested.
                    if (WaitForPandaRegistration)
                    {
                        var pandJobResult = Policy
                            .Handle<InvalidOperationException>()
                            .WaitAndRetryForever(nthRetry => TimeSpan.FromMinutes(1),
                                                 (e, ts) => { WriteWarning($"Failed to find the submitted panda job on bigpanda: {e.Message}. Will wait one minute and try again."); })
                            .ExecuteAndCapture(() => FindPandaJobForDS(ds));
                        if (pandJobResult.Outcome != OutcomeType.Successful)
                        {
                            throw pandJobResult.FinalException;
                        }
                        pandaJob = pandJobResult.Result;
                    }
                }

                // Return a helper obj that contains the info about this job that can be used by other commands.
                if (pandaJob != null)
                {
                    var r = new AtlasPandaTaskID() { ID = pandaJob.jeditaskid, Name = pandaJob.taskname };
                    WriteObject(r);
                }
            } finally
            {
                Trace.Listeners.Remove(listener);
            }
        }
예제 #6
0
 /// <summary>
 /// Init with the listener.
 /// </summary>
 /// <param name="listener"></param>
 public PSListenerPause(PSListener listener)
 {
     _listener          = listener;
     _origianlState     = listener.LogState;
     _listener.LogState = false;
 }
예제 #7
0
        /// <summary>
        /// Process a single dataset and fetch the info about it.
        /// </summary>
        protected override void ProcessRecord()
        {
            var cHit = _resultsCache.Value[DatasetName] as PSGRIDDatasetInfo;
            if (cHit != null)
            {
                WriteObject(cHit);
            }
            else
            {
                // Setup for verbosity if we need it.
                var listener = new PSListener(this);
                Trace.Listeners.Add(listener);
                try
                {
                    // Where are we going to be doing query on - we need a machine.
                    var sm = JobParser.GetSubmissionMachine();

                    // Get the remove environment configured if it needs to be
                    if (_connection == null)
                    {
                        _connection = new SSHConnection(sm.MachineName, sm.Username);
                        _connection
                            .Apply(() => DisplayStatus("Setting up ATLAS"))
                            .setupATLAS()
                            .Apply(() => DisplayStatus("Setting up Rucio"))
                            .setupRucio(_gridCredentials.Username)
                            .Apply(() => DisplayStatus("Acquiring GRID credentials"))
                            .VomsProxyInit("atlas", failNow: () => Stopping);
                    }

                    // Great - get the info on this dataset.
                    var fileInfo = _connection
                        .Apply(() => DisplayStatus($"Checking for info on {DatasetName}."))
                        .FileInfoFromGRID(DatasetName, failNow: () => Stopping);

                    // Next, build the resulting thingy.
                    var r = new PSGRIDDatasetInfo()
                    {
                        DatasetName = DatasetName,
                        nFiles = fileInfo.Count,
                        TotalSizeMB = (int)fileInfo.Sum(fi => fi.size),
                        FileInfo = fileInfo.ToArray()
                    };
                    _resultsCache.Value[DatasetName] = r;
                    WriteObject(r);
                }
                finally
                {
                    Trace.Listeners.Remove(listener);
                }
            }
        }