public ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableCohort extractableCohort, IExtractableDatasetBundle datasetBundle, List <IColumn> columnsToExtract, IHICProjectSalt salt, IExtractionDirectory directory, bool includeValidation = false, bool includeLookups = false) : this(configuration, datasetBundle.DataSet)
{
DataExportRepository = configuration.DataExportRepository;
ExtractableCohort = extractableCohort;
DatasetBundle = datasetBundle;
ColumnsToExtract = columnsToExtract;
Salt = salt;
Directory = directory;
IncludeValidation = includeValidation;
TopX = -1;
}
/// <summary>
/// Returns the object that was the source of the given <paramref name="cohort"/>.
/// May be a file or a <see cref="CohortIdentificationConfiguration"/> or other things or
/// null if its not possible to work out what created the cohort.
/// </summary>
/// <param name="cohort"></param>
/// <param name="repositoryLocator"></param>
/// <returns></returns>
public object GetObjectIfAny(IExtractableCohort cohort, Repositories.IRDMPPlatformRepositoryServiceLocator repositoryLocator)
{
var audit = cohort.AuditLog;
// no audit means no object
if (string.IsNullOrWhiteSpace(audit))
{
return(null);
}
if (_legacyCic.IsMatch(audit))
{
return(GetObjectFromLog <CohortIdentificationConfiguration>(_legacyCic.Match(audit), repositoryLocator.CatalogueRepository));
}
if (audit.Contains(InCohortIdentificationConfiguration))
{
return(GetObjectFromLog <CohortIdentificationConfiguration>(audit, repositoryLocator.CatalogueRepository));
}
if (audit.Contains(InExtractionInformation))
{
return(GetObjectFromLog <ExtractionInformation>(audit, repositoryLocator.CatalogueRepository));
}
if (audit.Contains(InFile))
{
var m = _regexGetFilePath.Match(audit);
if (m.Success)
{
try
{
return(new FileInfo(m.Groups[1].Value));
}
catch (System.Exception)
{
// its not a valid filename, maybe someone hacked the Audit Log by hand
return(null);
}
}
}
// who knows how this cohort was created
return(null);
}
public ExtractionTimeTimeCoverageAggregator(ICatalogue catalogue, IExtractableCohort cohort)
{
_catalogue = catalogue;
Buckets = new Dictionary <DateTime, ExtractionTimeTimeCoverageAggregatorBucket>();
if (!catalogue.TimeCoverage_ExtractionInformation_ID.HasValue)
{
return;
}
try
{
_expectedTimeFieldInOutputBuffer = catalogue.TimeCoverage_ExtractionInformation.GetRuntimeName();
}
catch (Exception e)
{
throw new Exception("Could not resolve TimeCoverage_ExtractionInformation for Catalogue '" + catalogue + "',time coverage ExtractionInformationID was:" + catalogue.TimeCoverage_ExtractionInformation_ID, e);
}
_expectedExtractionIdentifierInOutputBuffer = cohort.GetQuerySyntaxHelper().GetRuntimeName(cohort.GetReleaseIdentifier());
}
private void CopyCohortToDataServer(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
{
DataTable cohortDataTable = null;
SetServer();
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to wait for Semaphore OneCrossServerExtractionAtATime to become available"));
OneCrossServerExtractionAtATime.WaitOne(-1);
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Captured Semaphore OneCrossServerExtractionAtATime"));
try
{
IExtractableCohort cohort = Request.ExtractableCohort;
cohortDataTable = cohort.FetchEntireCohort();
}
catch (Exception e)
{
throw new Exception("An error occurred while trying to download the cohort from the Cohort server (in preparation for transfering it to the data server for linkage and extraction)", e);
}
//make sure tempdb exists (this covers you for servers where it doesn't exist e.g. mysql or when user has specified a different database name)
if (!_tempDb.Exists())
{
if (CreateAndDestroyTemporaryDatabaseIfNotExists)
{
_tempDb.Create();
_hadToCreate = true;
}
else
{
throw new Exception("Database '" + _tempDb + "' did not exist on server '" + _server + "' and CreateAndDestroyTemporaryDatabaseIfNotExists was false");
}
}
else
{
_hadToCreate = false;
}
var tbl = _tempDb.ExpectTable(cohortDataTable.TableName);
if (tbl.Exists())
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Found existing table called '" + tbl + "' in '" + _tempDb + "'"));
if (DropExistingCohortTableIfExists)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "About to drop existing table '" + tbl + "'"));
try
{
tbl.Drop();
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Dropped existing table '" + tbl + "'"));
}
catch (Exception ex)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Warning dropping '" + tbl + "' failed", ex));
}
}
else
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "'" + _tempDb + "' contains a table called '" + tbl + "' and DropExistingCohortTableIfExists is false"));
}
}
var destination = new DataTableUploadDestination();
destination.PreInitialize(_tempDb, listener);
destination.ProcessPipelineData(cohortDataTable, listener, cancellationToken);
destination.Dispose(listener, null);
if (!tbl.Exists())
{
throw new Exception("Table '" + tbl + "' did not exist despite DataTableUploadDestination completing Successfully!");
}
tablesToCleanup.Add(tbl);
//table will now be in tempdb
_haveCopiedCohortAndAdjustedSql = true;
}
public LinkedCohortNode(ExtractionConfiguration configuration, IExtractableCohort cohort)
{
Configuration = configuration;
Cohort = cohort;
}
public ReleaseIdentifierSubstitution(MemoryRepository repo, IColumn extractionIdentifierToSubFor, IExtractableCohort extractableCohort, bool isPartOfMultiCHISubstitution, IQuerySyntaxHelper querySyntaxHelper) : base(repo)
{
if (!extractionIdentifierToSubFor.IsExtractionIdentifier)
{
throw new Exception("Column " + extractionIdentifierToSubFor + " is not marked IsExtractionIdentifier so cannot be substituted for a ReleaseIdentifier");
}
OriginalDatasetColumn = extractionIdentifierToSubFor;
this._querySyntaxHelper = querySyntaxHelper;
if (OriginalDatasetColumn.ColumnInfo == null)
{
throw new Exception("The column " + OriginalDatasetColumn.GetRuntimeName() + " references a ColumnInfo that has been deleted");
}
var syntaxHelper = extractableCohort.GetQuerySyntaxHelper();
//the externally referenced Cohort table
var externalCohortTable = extractableCohort.ExternalCohortTable;
var privateIdentifierFieldDiscovered = externalCohortTable.Discover().ExpectTable(externalCohortTable.TableName).DiscoverColumn(externalCohortTable.PrivateIdentifierField);
string collateStatement = "";
//the release identifier join might require collation
//if the private has a collation
if (!string.IsNullOrWhiteSpace(privateIdentifierFieldDiscovered.Collation))
{
var cohortCollation = privateIdentifierFieldDiscovered.Collation;
var otherTableCollation = OriginalDatasetColumn.ColumnInfo.Collation;
//only collate if the server types match and if the collations differ
if (privateIdentifierFieldDiscovered.Table.Database.Server.DatabaseType == OriginalDatasetColumn.ColumnInfo.TableInfo.DatabaseType)
{
if (!string.IsNullOrWhiteSpace(otherTableCollation) && !string.Equals(cohortCollation, otherTableCollation))
{
collateStatement = " collate " + cohortCollation;
}
}
}
if (!isPartOfMultiCHISubstitution)
{
SelectSQL = extractableCohort.GetReleaseIdentifier();
Alias = syntaxHelper.GetRuntimeName(SelectSQL);
}
else
{
SelectSQL = "(SELECT DISTINCT " +
extractableCohort.GetReleaseIdentifier() +
" FROM " +
externalCohortTable.TableName + " WHERE " + extractableCohort.WhereSQL() + " AND " + externalCohortTable.PrivateIdentifierField + "=" + OriginalDatasetColumn.SelectSQL + collateStatement + ")";
if (!string.IsNullOrWhiteSpace(OriginalDatasetColumn.Alias))
{
string toReplace = extractableCohort.GetPrivateIdentifier(true);
string toReplaceWith = extractableCohort.GetReleaseIdentifier(true);
//take the same name as the underlying column
Alias = OriginalDatasetColumn.Alias;
//but replace all instances of CHI with PROCHI (or Barcode, or whatever)
if (!Alias.Contains(toReplace) || Regex.Matches(Alias, Regex.Escape(toReplace)).Count > 1)
{
throw new Exception("Expected OriginalDatasetColumn " + OriginalDatasetColumn.Alias + " to have the text \"" + toReplace + "\" appearing once (and only once in it's name)," +
"we planned to replace that text with:" + toReplaceWith);
}
Alias = Alias.Replace(toReplace, toReplaceWith);
}
else
{
throw new Exception("In cases where you have multiple columns marked IsExtractionIdentifier, they must all have Aliases, the column " + OriginalDatasetColumn.SelectSQL + " does not have one");
}
}
JoinSQL = OriginalDatasetColumn.SelectSQL + "=" + externalCohortTable.PrivateIdentifierField + collateStatement;
}
private void CreateExtractableCohort()
{
_extractableCohort = new ExtractableCohort(DataExportRepository, _externalCohortTable, cohortIDInTestData);
Assert.AreEqual(_extractableCohort.OriginID, cohortIDInTestData);
}
public static List <ConstantParameter> GetConstantParameters(IQuerySyntaxHelper syntaxHelper, IExtractionConfiguration configuration, IExtractableCohort extractableCohort)
{
List <ConstantParameter> toReturn = new List <ConstantParameter>();
if (syntaxHelper.DatabaseType == FAnsi.DatabaseType.Oracle)
{
return(toReturn);
}
IProject project = configuration.Project;
if (project.ProjectNumber == null)
{
throw new ProjectNumberException("Project number has not been entered, cannot create constant paramaters");
}
if (extractableCohort == null)
{
throw new Exception("Cohort has not been selected, cannot create constant parameters");
}
IExternalCohortTable externalCohortTable = extractableCohort.ExternalCohortTable;
var declarationSqlCohortId = syntaxHelper.GetParameterDeclaration("@CohortDefinitionID", new DatabaseTypeRequest(typeof(int)));
var declarationSqlProjectNumber = syntaxHelper.GetParameterDeclaration("@ProjectNumber", new DatabaseTypeRequest(typeof(int)));
toReturn.Add(new ConstantParameter(declarationSqlCohortId, extractableCohort.OriginID.ToString(), "The ID of the cohort in " + externalCohortTable.TableName, syntaxHelper));
toReturn.Add(new ConstantParameter(declarationSqlProjectNumber, project.ProjectNumber.ToString(), "The project number of project " + project.Name, syntaxHelper));
return(toReturn);
}
