private void WriteBundleContents(IExtractableDatasetBundle datasetBundle, IDataLoadEventListener job, GracefulCancellationToken cancellationToken)
{
var rootDir = _request.GetExtractionDirectory();
var supportingSQLFolder = new DirectoryInfo(Path.Combine(rootDir.FullName, SupportingSQLTable.ExtractionFolderName));
var lookupDir = rootDir.CreateSubdirectory("Lookups");
//extract the documents
foreach (SupportingDocument doc in datasetBundle.Documents)
{
datasetBundle.States[doc] = TryExtractSupportingDocument(doc, rootDir, job)
? ExtractCommandState.Completed
: ExtractCommandState.Crashed;
}
//extract supporting SQL
foreach (SupportingSQLTable sql in datasetBundle.SupportingSQL)
{
datasetBundle.States[sql] = TryExtractSupportingSQLTable(sql, supportingSQLFolder, _request.Configuration, job, _dataLoadInfo)
? ExtractCommandState.Completed
: ExtractCommandState.Crashed;
}
//extract lookups
foreach (BundledLookupTable lookup in datasetBundle.LookupTables)
{
datasetBundle.States[lookup] = TryExtractLookupTable(lookup, lookupDir, job)
? ExtractCommandState.Completed
: ExtractCommandState.Crashed;
}
}
private void WriteBundleContents(IExtractableDatasetBundle datasetBundle, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
{
var bundle = ((ExtractDatasetCommand)_request).DatasetBundle;
foreach (var sql in bundle.SupportingSQL)
{
bundle.States[sql] = ExtractSupportingSql(sql, listener, _dataLoadInfo);
}
foreach (var document in ((ExtractDatasetCommand)_request).DatasetBundle.Documents)
{
bundle.States[document] = ExtractSupportingDocument(_request.GetExtractionDirectory(), document, listener);
}
//extract lookups
foreach (BundledLookupTable lookup in datasetBundle.LookupTables)
{
try
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to extract lookup " + lookup));
ExtractLookupTableSql(lookup, listener, _dataLoadInfo);
datasetBundle.States[lookup] = ExtractCommandState.Completed;
}
catch (Exception e)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Error occurred trying to extract lookup " + lookup + " on server " + lookup.TableInfo.Server, e));
datasetBundle.States[lookup] = ExtractCommandState.Crashed;
}
}
haveExtractedBundledContent = true;
}
private void WriteBundleContents(IExtractableDatasetBundle datasetBundle, IDataLoadEventListener job, GracefulCancellationToken cancellationToken)
{
var rootDir = _request.GetExtractionDirectory();
var supportingSQLFolder = new DirectoryInfo(Path.Combine(rootDir.FullName, SupportingSQLTable.ExtractionFolderName));
var lookupDir = rootDir.CreateSubdirectory("Lookups");
//extract the documents
foreach (SupportingDocument doc in datasetBundle.Documents)
{
datasetBundle.States[doc] = TryExtractSupportingDocument(rootDir, doc, job)
? ExtractCommandState.Completed
: ExtractCommandState.Crashed;
}
//extract supporting SQL
foreach (SupportingSQLTable sql in datasetBundle.SupportingSQL)
{
datasetBundle.States[sql] = TryExtractSupportingSQLTable(supportingSQLFolder, _request.Configuration, sql, job, _dataLoadInfo)
? ExtractCommandState.Completed
: ExtractCommandState.Crashed;
}
//extract lookups
foreach (BundledLookupTable lookup in datasetBundle.LookupTables)
{
try
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to extract lookup " + lookup));
var server = DataAccessPortal.GetInstance().ExpectServer(lookup.TableInfo, DataAccessContext.DataExport);
Stopwatch sw = new Stopwatch();
sw.Start();
//extracts all of them
var extractTableVerbatim = new ExtractTableVerbatim(server, new [] { lookup.TableInfo.Name }, lookupDir, _request.Configuration.Separator, DateFormat);
int linesWritten = extractTableVerbatim.DoExtraction();
sw.Stop();
job.OnProgress(this, new ProgressEventArgs("Lookup " + lookup, new ProgressMeasurement(linesWritten, ProgressType.Records), sw.Elapsed));
if (_request is ExtractDatasetCommand)
{
var result = (_request as ExtractDatasetCommand).CumulativeExtractionResults;
var supplementalResult = result.AddSupplementalExtractionResult("SELECT * FROM " + lookup.TableInfo.Name, lookup.TableInfo);
supplementalResult.CompleteAudit(this.GetType(), extractTableVerbatim.OutputFilename, linesWritten);
}
datasetBundle.States[lookup] = ExtractCommandState.Completed;
}
catch (Exception e)
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Error occurred trying to extract lookup " + lookup + " on server " + lookup.TableInfo.Server, e));
datasetBundle.States[lookup] = ExtractCommandState.Crashed;
}
}
haveWrittenBundleContents = true;
}
public ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableCohort extractableCohort, IExtractableDatasetBundle datasetBundle, List <IColumn> columnsToExtract, IHICProjectSalt salt, IExtractionDirectory directory, bool includeValidation = false, bool includeLookups = false) : this(configuration, datasetBundle.DataSet)
{
DataExportRepository = configuration.DataExportRepository;
ExtractableCohort = extractableCohort;
DatasetBundle = datasetBundle;
ColumnsToExtract = columnsToExtract;
Salt = salt;
Directory = directory;
IncludeValidation = includeValidation;
TopX = -1;
}
/// <summary>
/// This version has less arguments because it goes back to the database and queries the configuration and explores who the cohort is etc, it will result in more database
/// queries than the more explicit constructor
/// </summary>
/// <param name="configuration"></param>
/// <param name="datasetBundle"></param>
/// <param name="includeValidation"></param>
/// <param name="includeLookups"></param>
public ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableDatasetBundle datasetBundle, bool includeValidation = false, bool includeLookups = false) : this(configuration, datasetBundle.DataSet)
{
DataExportRepository = configuration.DataExportRepository;
//ExtractableCohort = ExtractableCohort.GetExtractableCohortByID((int) configuration.Cohort_ID);
ExtractableCohort = configuration.GetExtractableCohort();
DatasetBundle = datasetBundle;
ColumnsToExtract = new List <IColumn>(Configuration.GetAllExtractableColumnsFor(datasetBundle.DataSet));
Salt = new HICProjectSalt(Project);
Directory = new ExtractionDirectory(Project.ExtractionDirectory, configuration);
IncludeValidation = includeValidation;
TopX = -1;
}