Esempio n. 1
0
        /// <summary>
        /// Load up the job requested. Fail, obviously, if we can't.
        /// </summary>
        protected override void ProcessRecord()
        {
            // Setup for verbosity if we need it.
            var listener = new PSListener(this);
            Trace.Listeners.Add(listener);
            try
            {
                // Get the job
                var job = JobParser.FindJob(JobName, JobVersion);

                // Get the expected resulting dataset name. Since this will be a personal
                // dataset, we need to get the GRID info.
                string ds = job.ResultingDatasetName(DatasetName, _gridCredentials);

                // See if there is already a job defined that will produce this
                var pandaJob = (ds + "/").FindPandaJobWithTaskName();

                if (pandaJob == null)
                {
                    // Where are we going to be doing the submission on?
                    var sm = JobParser.GetSubmissionMachine();

                    // Get the remove environment configured if it needs to be
                    var firstJob = false;
                    if (_connection == null)
                    {
                        firstJob = true;
                        _connection = new SSHConnection(sm.MachineName, sm.Username);
                        _connection
                            .Apply(() => DisplayStatus("Setting up ATLAS"))
                            .setupATLAS(dumpOnly: WhatIf.IsPresent)
                            .Apply(() => DisplayStatus("Setting up Rucio"))
                            .setupRucio(_gridCredentials.Username, dumpOnly: WhatIf.IsPresent)
                            .Apply(() => DisplayStatus("Acquiring GRID credentials"))
                            .VomsProxyInit("atlas", failNow: () => Stopping, dumpOnly: WhatIf.IsPresent);
                    }

                    // Check to see if the original dataset exists. We will use the location known as Local for doing the
                    // setup, I suppose.
                    var files = _connection
                        .Apply(() => DisplayStatus("Checking dataset exists on the GRID"))
                        .FilelistFromGRID(DatasetName, failNow: () => Stopping, dumpOnly: WhatIf.IsPresent);
                    if (files.Length == 0 && !WhatIf.IsPresent)
                    {
                        throw new ArgumentException($"Dataset '{DatasetName}' has zero files - won't submit a job against it!");
                    }

                    // Submit the job
                    _connection
                        .SubmitJob(job, DatasetName, ds, DisplayStatus, failNow: () => Stopping, sameJobAsLastTime: !firstJob, dumpOnly: WhatIf.IsPresent);

                    // Try to find the job again if requested. The submission can take a very long time to show up in
                    // big panda, so skip unless requested.
                    if (WaitForPandaRegistration)
                    {
                        var pandJobResult = Policy
                            .Handle<InvalidOperationException>()
                            .WaitAndRetryForever(nthRetry => TimeSpan.FromMinutes(1),
                                                 (e, ts) => { WriteWarning($"Failed to find the submitted panda job on bigpanda: {e.Message}. Will wait one minute and try again."); })
                            .ExecuteAndCapture(() => FindPandaJobForDS(ds));
                        if (pandJobResult.Outcome != OutcomeType.Successful)
                        {
                            throw pandJobResult.FinalException;
                        }
                        pandaJob = pandJobResult.Result;
                    }
                }

                // Return a helper obj that contains the info about this job that can be used by other commands.
                if (pandaJob != null)
                {
                    var r = new AtlasPandaTaskID() { ID = pandaJob.jeditaskid, Name = pandaJob.taskname };
                    WriteObject(r);
                }
            } finally
            {
                Trace.Listeners.Remove(listener);
            }
        }
Esempio n. 2
0
        /// <summary>
        /// Create the connection, and setup the appropriate stuff for using AMI.
        /// </summary>
        /// <returns></returns>
        private SSHConnection GetConnection()
        {
            if (_connection != null)
                return _connection;

            // Establish the connection
            var sm = JobParser.GetSubmissionMachine();
            var connection = new SSHConnection(sm.MachineName, sm.Username);

            // Setup for using ami.
            connection.Apply(() => DisplayStatus("Setting up ATLAS"))
                .setupATLAS()
                .Apply(() => DisplayStatus("Setting up pyAMI"))
                .ExecuteCommand("lsetup pyami")
                .Apply(() => DisplayStatus("Acquiring GRID credentials"))
                .VomsProxyInit("atlas", failNow: () => Stopping);

            // And the connection is now ready for use!
            _connection = connection;
            return _connection;
        }
Esempio n. 3
0
        /// <summary>
        /// Process a single dataset and fetch the info about it.
        /// </summary>
        protected override void ProcessRecord()
        {
            var cHit = _resultsCache.Value[DatasetName] as PSGRIDDatasetInfo;
            if (cHit != null)
            {
                WriteObject(cHit);
            }
            else
            {
                // Setup for verbosity if we need it.
                var listener = new PSListener(this);
                Trace.Listeners.Add(listener);
                try
                {
                    // Where are we going to be doing query on - we need a machine.
                    var sm = JobParser.GetSubmissionMachine();

                    // Get the remove environment configured if it needs to be
                    if (_connection == null)
                    {
                        _connection = new SSHConnection(sm.MachineName, sm.Username);
                        _connection
                            .Apply(() => DisplayStatus("Setting up ATLAS"))
                            .setupATLAS()
                            .Apply(() => DisplayStatus("Setting up Rucio"))
                            .setupRucio(_gridCredentials.Username)
                            .Apply(() => DisplayStatus("Acquiring GRID credentials"))
                            .VomsProxyInit("atlas", failNow: () => Stopping);
                    }

                    // Great - get the info on this dataset.
                    var fileInfo = _connection
                        .Apply(() => DisplayStatus($"Checking for info on {DatasetName}."))
                        .FileInfoFromGRID(DatasetName, failNow: () => Stopping);

                    // Next, build the resulting thingy.
                    var r = new PSGRIDDatasetInfo()
                    {
                        DatasetName = DatasetName,
                        nFiles = fileInfo.Count,
                        TotalSizeMB = (int)fileInfo.Sum(fi => fi.size),
                        FileInfo = fileInfo.ToArray()
                    };
                    _resultsCache.Value[DatasetName] = r;
                    WriteObject(r);
                }
                finally
                {
                    Trace.Listeners.Remove(listener);
                }
            }
        }