public ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableCohort extractableCohort, IExtractableDatasetBundle datasetBundle, List <IColumn> columnsToExtract, IHICProjectSalt salt, IExtractionDirectory directory, bool includeValidation = false, bool includeLookups = false) : this(configuration, datasetBundle.DataSet) { DataExportRepository = configuration.DataExportRepository; ExtractableCohort = extractableCohort; DatasetBundle = datasetBundle; ColumnsToExtract = columnsToExtract; Salt = salt; Directory = directory; IncludeValidation = includeValidation; TopX = -1; }
/// <summary> /// Returns the object that was the source of the given <paramref name="cohort"/>. /// May be a file or a <see cref="CohortIdentificationConfiguration"/> or other things or /// null if its not possible to work out what created the cohort. /// </summary> /// <param name="cohort"></param> /// <param name="repositoryLocator"></param> /// <returns></returns> public object GetObjectIfAny(IExtractableCohort cohort, Repositories.IRDMPPlatformRepositoryServiceLocator repositoryLocator) { var audit = cohort.AuditLog; // no audit means no object if (string.IsNullOrWhiteSpace(audit)) { return(null); } if (_legacyCic.IsMatch(audit)) { return(GetObjectFromLog <CohortIdentificationConfiguration>(_legacyCic.Match(audit), repositoryLocator.CatalogueRepository)); } if (audit.Contains(InCohortIdentificationConfiguration)) { return(GetObjectFromLog <CohortIdentificationConfiguration>(audit, repositoryLocator.CatalogueRepository)); } if (audit.Contains(InExtractionInformation)) { return(GetObjectFromLog <ExtractionInformation>(audit, repositoryLocator.CatalogueRepository)); } if (audit.Contains(InFile)) { var m = _regexGetFilePath.Match(audit); if (m.Success) { try { return(new FileInfo(m.Groups[1].Value)); } catch (System.Exception) { // its not a valid filename, maybe someone hacked the Audit Log by hand return(null); } } } // who knows how this cohort was created return(null); }
public ExtractionTimeTimeCoverageAggregator(ICatalogue catalogue, IExtractableCohort cohort) { _catalogue = catalogue; Buckets = new Dictionary <DateTime, ExtractionTimeTimeCoverageAggregatorBucket>(); if (!catalogue.TimeCoverage_ExtractionInformation_ID.HasValue) { return; } try { _expectedTimeFieldInOutputBuffer = catalogue.TimeCoverage_ExtractionInformation.GetRuntimeName(); } catch (Exception e) { throw new Exception("Could not resolve TimeCoverage_ExtractionInformation for Catalogue '" + catalogue + "',time coverage ExtractionInformationID was:" + catalogue.TimeCoverage_ExtractionInformation_ID, e); } _expectedExtractionIdentifierInOutputBuffer = cohort.GetQuerySyntaxHelper().GetRuntimeName(cohort.GetReleaseIdentifier()); }
private void CopyCohortToDataServer(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { DataTable cohortDataTable = null; SetServer(); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to wait for Semaphore OneCrossServerExtractionAtATime to become available")); OneCrossServerExtractionAtATime.WaitOne(-1); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Captured Semaphore OneCrossServerExtractionAtATime")); try { IExtractableCohort cohort = Request.ExtractableCohort; cohortDataTable = cohort.FetchEntireCohort(); } catch (Exception e) { throw new Exception("An error occurred while trying to download the cohort from the Cohort server (in preparation for transfering it to the data server for linkage and extraction)", e); } //make sure tempdb exists (this covers you for servers where it doesn't exist e.g. mysql or when user has specified a different database name) if (!_tempDb.Exists()) { if (CreateAndDestroyTemporaryDatabaseIfNotExists) { _tempDb.Create(); _hadToCreate = true; } else { throw new Exception("Database '" + _tempDb + "' did not exist on server '" + _server + "' and CreateAndDestroyTemporaryDatabaseIfNotExists was false"); } } else { _hadToCreate = false; } var tbl = _tempDb.ExpectTable(cohortDataTable.TableName); if (tbl.Exists()) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Found existing table called '" + tbl + "' in '" + _tempDb + "'")); if (DropExistingCohortTableIfExists) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "About to drop existing table '" + tbl + "'")); try { tbl.Drop(); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Dropped existing table '" + tbl + "'")); } catch (Exception ex) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Warning dropping '" + tbl + "' failed", ex)); } } else { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "'" + _tempDb + "' contains a table called '" + tbl + "' and DropExistingCohortTableIfExists is false")); } } var destination = new DataTableUploadDestination(); destination.PreInitialize(_tempDb, listener); destination.ProcessPipelineData(cohortDataTable, listener, cancellationToken); destination.Dispose(listener, null); if (!tbl.Exists()) { throw new Exception("Table '" + tbl + "' did not exist despite DataTableUploadDestination completing Successfully!"); } tablesToCleanup.Add(tbl); //table will now be in tempdb _haveCopiedCohortAndAdjustedSql = true; }
public LinkedCohortNode(ExtractionConfiguration configuration, IExtractableCohort cohort) { Configuration = configuration; Cohort = cohort; }
public ReleaseIdentifierSubstitution(MemoryRepository repo, IColumn extractionIdentifierToSubFor, IExtractableCohort extractableCohort, bool isPartOfMultiCHISubstitution, IQuerySyntaxHelper querySyntaxHelper) : base(repo) { if (!extractionIdentifierToSubFor.IsExtractionIdentifier) { throw new Exception("Column " + extractionIdentifierToSubFor + " is not marked IsExtractionIdentifier so cannot be substituted for a ReleaseIdentifier"); } OriginalDatasetColumn = extractionIdentifierToSubFor; this._querySyntaxHelper = querySyntaxHelper; if (OriginalDatasetColumn.ColumnInfo == null) { throw new Exception("The column " + OriginalDatasetColumn.GetRuntimeName() + " references a ColumnInfo that has been deleted"); } var syntaxHelper = extractableCohort.GetQuerySyntaxHelper(); //the externally referenced Cohort table var externalCohortTable = extractableCohort.ExternalCohortTable; var privateIdentifierFieldDiscovered = externalCohortTable.Discover().ExpectTable(externalCohortTable.TableName).DiscoverColumn(externalCohortTable.PrivateIdentifierField); string collateStatement = ""; //the release identifier join might require collation //if the private has a collation if (!string.IsNullOrWhiteSpace(privateIdentifierFieldDiscovered.Collation)) { var cohortCollation = privateIdentifierFieldDiscovered.Collation; var otherTableCollation = OriginalDatasetColumn.ColumnInfo.Collation; //only collate if the server types match and if the collations differ if (privateIdentifierFieldDiscovered.Table.Database.Server.DatabaseType == OriginalDatasetColumn.ColumnInfo.TableInfo.DatabaseType) { if (!string.IsNullOrWhiteSpace(otherTableCollation) && !string.Equals(cohortCollation, otherTableCollation)) { collateStatement = " collate " + cohortCollation; } } } if (!isPartOfMultiCHISubstitution) { SelectSQL = extractableCohort.GetReleaseIdentifier(); Alias = syntaxHelper.GetRuntimeName(SelectSQL); } else { SelectSQL = "(SELECT DISTINCT " + extractableCohort.GetReleaseIdentifier() + " FROM " + externalCohortTable.TableName + " WHERE " + extractableCohort.WhereSQL() + " AND " + externalCohortTable.PrivateIdentifierField + "=" + OriginalDatasetColumn.SelectSQL + collateStatement + ")"; if (!string.IsNullOrWhiteSpace(OriginalDatasetColumn.Alias)) { string toReplace = extractableCohort.GetPrivateIdentifier(true); string toReplaceWith = extractableCohort.GetReleaseIdentifier(true); //take the same name as the underlying column Alias = OriginalDatasetColumn.Alias; //but replace all instances of CHI with PROCHI (or Barcode, or whatever) if (!Alias.Contains(toReplace) || Regex.Matches(Alias, Regex.Escape(toReplace)).Count > 1) { throw new Exception("Expected OriginalDatasetColumn " + OriginalDatasetColumn.Alias + " to have the text \"" + toReplace + "\" appearing once (and only once in it's name)," + "we planned to replace that text with:" + toReplaceWith); } Alias = Alias.Replace(toReplace, toReplaceWith); } else { throw new Exception("In cases where you have multiple columns marked IsExtractionIdentifier, they must all have Aliases, the column " + OriginalDatasetColumn.SelectSQL + " does not have one"); } } JoinSQL = OriginalDatasetColumn.SelectSQL + "=" + externalCohortTable.PrivateIdentifierField + collateStatement; }
private void CreateExtractableCohort() { _extractableCohort = new ExtractableCohort(DataExportRepository, _externalCohortTable, cohortIDInTestData); Assert.AreEqual(_extractableCohort.OriginID, cohortIDInTestData); }
public static List <ConstantParameter> GetConstantParameters(IQuerySyntaxHelper syntaxHelper, IExtractionConfiguration configuration, IExtractableCohort extractableCohort) { List <ConstantParameter> toReturn = new List <ConstantParameter>(); if (syntaxHelper.DatabaseType == FAnsi.DatabaseType.Oracle) { return(toReturn); } IProject project = configuration.Project; if (project.ProjectNumber == null) { throw new ProjectNumberException("Project number has not been entered, cannot create constant paramaters"); } if (extractableCohort == null) { throw new Exception("Cohort has not been selected, cannot create constant parameters"); } IExternalCohortTable externalCohortTable = extractableCohort.ExternalCohortTable; var declarationSqlCohortId = syntaxHelper.GetParameterDeclaration("@CohortDefinitionID", new DatabaseTypeRequest(typeof(int))); var declarationSqlProjectNumber = syntaxHelper.GetParameterDeclaration("@ProjectNumber", new DatabaseTypeRequest(typeof(int))); toReturn.Add(new ConstantParameter(declarationSqlCohortId, extractableCohort.OriginID.ToString(), "The ID of the cohort in " + externalCohortTable.TableName, syntaxHelper)); toReturn.Add(new ConstantParameter(declarationSqlProjectNumber, project.ProjectNumber.ToString(), "The project number of project " + project.Name, syntaxHelper)); return(toReturn); }