Пример #1
0
        /// <summary>
        /// Deal with any arguments and possible output filename.
        /// If a output filename is specified, create a string 'format' based
        /// on that and string out the arg.
        /// </summary>
        private void ParseExecutableArgs(AnalysisSpecification analysisSpec, AnalysisEntity analysisEntity)
        {
            var outputFormat   = "output-{0}.xml";
            var executableArgs = analysisSpec.ExecutableArgs;

            if (!string.IsNullOrEmpty(executableArgs))
            {
                var tokens = executableArgs.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries).ToList();

                if (tokens.Any(token => token == "-out"))
                {
                    var indexOfOutArg      = tokens.IndexOf("-out");
                    var indexOfOutFilename = indexOfOutArg + 1;
                    if (indexOfOutFilename > tokens.Count - 1)
                    {
                        throw new Exception("No filename specified for -out argument");
                    }
                    var outputFilename = tokens[indexOfOutArg + 1];
                    var name           = Path.GetFileNameWithoutExtension(outputFilename);
                    var extension      = Path.GetExtension(outputFilename);
                    outputFormat = name + "-{0}" + extension;

                    // Strip out the -out filename args as we already set it later
                    executableArgs = executableArgs.Replace(" -out " + outputFilename, " ");
                }
            }

            analysisEntity.ExecutableArgsSanitised = executableArgs;
            analysisEntity.OutputfileFormat        = outputFormat;
        }
Пример #2
0
        private AnalysisEntity CreateAnalysisEntity(AnalysisSpecification analysis, DatabaseEntity database)
        {
            var queryId        = Guid.NewGuid();
            var analysisEntity = new AnalysisEntity(queryId);

            analysisEntity.Name                    = analysis.Name;
            analysisEntity.JobId                   = queryId.ToString();
            analysisEntity.InputContainer          = queryId.ToString();
            analysisEntity.OutputContainer         = queryId.ToString();
            analysisEntity.DatabaseId              = analysis.DatabaseName;
            analysisEntity.DatabaseType            = database.Type;
            analysisEntity.DatabaseContainer       = database.ContainerName;
            analysisEntity.Executable              = analysis.Executable;
            analysisEntity.ExecutableArgs          = analysis.ExecutableArgs;
            analysisEntity.ExecutableArgsSanitised = analysis.ExecutableArgs;
            analysisEntity.State                   = AnalysisState.StagingData;
            analysisEntity.StartTime               = DateTime.UtcNow;
            analysisEntity.PoolId                  = analysis.PoolId;
            analysisEntity.PoolDisplayName         = analysis.PoolDisplayName;
            analysisEntity.TargetDedicated         = analysis.TargetDedicated;
            analysisEntity.VirtualMachineSize      = analysis.VirtualMachineSize;
            analysisEntity.CompletedTasks          = 0;
            analysisEntity.TotalTasks              = analysis.AnalysisInputFiles.Count();
            ParseExecutableArgs(analysis, analysisEntity);
            _tableStorageProvider.InsertEntity(analysisEntity);
            return(analysisEntity);
        }
Пример #3
0
        public Guid SubmitAnalysis(AnalysisSpecification analysis)
        {
            if (analysis == null)
            {
                throw new ArgumentNullException("analysis");
            }

            ValidateExecutableArgs(analysis.ExecutableArgs);

            var db = _databaseProvider.GetDatabase(analysis.DatabaseName);

            if (db == null)
            {
                throw new ArgumentException(string.Format("Cannot find database {0}", analysis.DatabaseName));
            }
            var fragments = _databaseProvider.GetDatabaseFragments(analysis.DatabaseName);

            if (fragments == null || fragments.Count == 0)
            {
                throw new ArgumentException(string.Format("Database has no fragments {0}", analysis.DatabaseName));
            }

            var analysisEntity = CreateAnalysisEntity(analysis, db);

            try
            {
                // Upload all the inputs to storage
                Parallel.ForEach(analysis.AnalysisInputFiles, queryFile =>
                {
                    var filename = Path.GetFileName(queryFile.Filename);
                    _blobStorageProvider.UploadBlobFromStream(analysisEntity.InputContainer, filename, queryFile.Content);
                });

                var queryIndex      = 0;
                var analysisQueries = new List <AnalysisQueryEntity>();
                foreach (var queryFile in analysis.AnalysisInputFiles)
                {
                    var query = new AnalysisQueryEntity(analysisEntity.Id, queryIndex.ToString());
                    query.OutputContainer     = analysisEntity.OutputContainer;
                    query.QueryFilename       = Path.GetFileName(queryFile.Filename);
                    query.State               = QueryState.Waiting;
                    query.QueryOutputFilename = GetQueryOutputFilename(analysisEntity.OutputfileFormat, queryIndex.ToString());
                    query.LogOutputFilename   = GetLogFilename(analysisEntity.OutputfileFormat, queryIndex.ToString());
                    analysisQueries.Add(query);
                    queryIndex++;
                }

                _tableStorageProvider.InsertEntities(analysisQueries);

                // Stage the generic batch scripts to storage
                var resourceFiles = InputFileStager.StageImportScripts(_blobStorageProvider);
                SubmitBatchJob(analysisEntity, analysisQueries, resourceFiles);

                analysisEntity.State = AnalysisState.WaitingForResources;
                _tableStorageProvider.UpdateEntity(analysisEntity);
            }
            catch (Exception e)
            {
                if (e is AggregateException)
                {
                    e = e.InnerException;
                }

                analysisEntity.State  = AnalysisState.Error;
                analysisEntity.Errors = e.ToString();
                _tableStorageProvider.UpdateEntity(analysisEntity);

                throw e;
            }

            return(analysisEntity.Id);
        }