public void SubmitImport(
ExternalRepository externalRepository,
ExternalDatabase externalDatabase)
{
ValidateNoImportsInProgress(externalDatabase.Name);
var jobId = string.Format("import-{0}-{1}", externalDatabase.Name, Guid.NewGuid());
var fragments = externalRepository.DatabaseSource.GetDatabaseFragments(externalDatabase.Name);
var entity = CreateDatabaseEntity(SystemDatabaseProvider.DefaultContainerName, jobId, externalDatabase.Name, fragments.Count);
try
{
// Stage all the job scripts
var resourceFiles = InputFileStager.StageImportScripts(_blobStorageProvider);
// Submit the job and tasks to batch
SubmitBatchJob(jobId, externalDatabase.Name, fragments, resourceFiles);
}
catch (Exception)
{
CleanupAfterFailure(jobId, entity);
throw;
}
}
public Guid SubmitAnalysis(AnalysisSpecification analysis)
{
if (analysis == null)
{
throw new ArgumentNullException("analysis");
}
ValidateExecutableArgs(analysis.ExecutableArgs);
var db = _databaseProvider.GetDatabase(analysis.DatabaseName);
if (db == null)
{
throw new ArgumentException(string.Format("Cannot find database {0}", analysis.DatabaseName));
}
var fragments = _databaseProvider.GetDatabaseFragments(analysis.DatabaseName);
if (fragments == null || fragments.Count == 0)
{
throw new ArgumentException(string.Format("Database has no fragments {0}", analysis.DatabaseName));
}
var analysisEntity = CreateAnalysisEntity(analysis, db);
try
{
// Upload all the inputs to storage
Parallel.ForEach(analysis.AnalysisInputFiles, queryFile =>
{
var filename = Path.GetFileName(queryFile.Filename);
_blobStorageProvider.UploadBlobFromStream(analysisEntity.InputContainer, filename, queryFile.Content);
});
var queryIndex = 0;
var analysisQueries = new List <AnalysisQueryEntity>();
foreach (var queryFile in analysis.AnalysisInputFiles)
{
var query = new AnalysisQueryEntity(analysisEntity.Id, queryIndex.ToString());
query.OutputContainer = analysisEntity.OutputContainer;
query.QueryFilename = Path.GetFileName(queryFile.Filename);
query.State = QueryState.Waiting;
query.QueryOutputFilename = GetQueryOutputFilename(analysisEntity.OutputfileFormat, queryIndex.ToString());
query.LogOutputFilename = GetLogFilename(analysisEntity.OutputfileFormat, queryIndex.ToString());
analysisQueries.Add(query);
queryIndex++;
}
_tableStorageProvider.InsertEntities(analysisQueries);
// Stage the generic batch scripts to storage
var resourceFiles = InputFileStager.StageImportScripts(_blobStorageProvider);
SubmitBatchJob(analysisEntity, analysisQueries, resourceFiles);
analysisEntity.State = AnalysisState.WaitingForResources;
_tableStorageProvider.UpdateEntity(analysisEntity);
}
catch (Exception e)
{
if (e is AggregateException)
{
e = e.InnerException;
}
analysisEntity.State = AnalysisState.Error;
analysisEntity.Errors = e.ToString();
_tableStorageProvider.UpdateEntity(analysisEntity);
throw e;
}
return(analysisEntity.Id);
}