public void DodgyCharactersInCatalogueName()
{
string beforeName = _catalogue.Name;
try
{
_catalogue.Name = "Fish;#:::FishFish";
Assert.IsFalse(Catalogue.IsAcceptableName(_catalogue.Name));
_catalogue.SaveToDatabase();
_extractableDataSet.RevertToDatabaseState();
var extractionDirectory = new ExtractionDirectory(TestContext.CurrentContext.WorkDirectory, _configuration);
var ex = Assert.Throws <NotSupportedException>(() => { var dir = extractionDirectory.GetDirectoryForDataset(_extractableDataSet); });
Assert.AreEqual("Cannot extract dataset Fish;#:::FishFish because it points at Catalogue with an invalid name, name is invalid because:The following invalid characters were found:'#'", ex.Message);
}
finally
{
_catalogue.Name = beforeName;
_catalogue.SaveToDatabase();
_extractableDataSet.RevertToDatabaseState();
}
}
private void CheckReleaseConfiguration(ICheckNotifier notifier)
{
var projectDirectory = new DirectoryInfo(_config.Project.ExtractionDirectory);
notifier.OnCheckPerformed(new CheckEventArgs("Found Frozen/Released configuration '" + _config + "'", CheckResult.Success));
foreach (DirectoryInfo directoryInfo in projectDirectory.GetDirectories(ExtractionDirectory.GetExtractionDirectoryPrefix(_config) + "*").ToArray())
{
string firstFileFound;
if (DirectoryIsEmpty(directoryInfo, out firstFileFound))
{
bool deleteIt =
notifier.OnCheckPerformed(
new CheckEventArgs(
"Found empty folder " + directoryInfo.Name +
" which is left over extracted folder after data release", CheckResult.Warning, null,
"Delete empty folder"));
if (deleteIt)
{
directoryInfo.Delete(true);
}
}
else
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Found non-empty folder " + directoryInfo.Name +
" which is left over extracted folder after data release (First file found was '" + firstFileFound + "' but there may be others)", CheckResult.Fail));
}
}
}
/// <summary>
/// Checks extraction directory for unexpected files. Call <see cref="GetEvaluator"/> if want to know whether a specific global artifact is in a valid
/// state.
/// </summary>
/// <param name="notifier"></param>
public void Check(ICheckNotifier notifier)
{
// checks for pollution in the globals directory
foreach (var extractionConfiguration in _configurations)
{
var allExtracted = extractionConfiguration.SupplementalExtractionResults.Where(ser => IsValidPath(ser.DestinationDescription));
var extractDir = extractionConfiguration.GetProject().ExtractionDirectory;
var folder = new ExtractionDirectory(extractDir, extractionConfiguration).GetGlobalsDirectory();
var unexpectedDirectories = folder.EnumerateDirectories().Where(d => !d.Name.Equals("SupportingDocuments")).ToList();
if (unexpectedDirectories.Any())
{
notifier.OnCheckPerformed(new CheckEventArgs("Unexpected directories found in extraction directory (" +
String.Join(",", unexpectedDirectories.Select(d => d.FullName)) +
". Pollution of extract directory is not permitted.", CheckResult.Fail));
}
var unexpectedFiles = folder.EnumerateFiles("*.*", SearchOption.AllDirectories).Where(f => allExtracted.All(ae => ae.DestinationDescription != f.FullName)).ToList();
if (unexpectedFiles.Any())
{
notifier.OnCheckPerformed(new CheckEventArgs("Unexpected files found in extract directory (" +
String.Join(",", unexpectedFiles.Select(d => d.FullName)) +
"). Pollution of extract directory is not permitted.", CheckResult.Fail));
}
}
}
protected override Releaseability GetSpecificAssessment(IExtractionResults extractionResults)
{
var _extractDir = Configuration.GetProject().ExtractionDirectory;
ExtractDirectory = new ExtractionDirectory(_extractDir, Configuration).GetDirectoryForDataset(DataSet);
var externalServerId = int.Parse(extractionResults.DestinationDescription.Split('|')[0]);
var externalServer = _repositoryLocator.CatalogueRepository.GetObjectByID <ExternalDatabaseServer>(externalServerId);
var dbName = extractionResults.DestinationDescription.Split('|')[1];
var tblName = extractionResults.DestinationDescription.Split('|')[2];
var server = DataAccessPortal.GetInstance().ExpectServer(externalServer, DataAccessContext.DataExport, setInitialDatabase: false);
var database = server.ExpectDatabase(dbName);
if (!database.Exists())
{
return(Releaseability.ExtractFilesMissing);
}
var foundTable = database.ExpectTable(tblName);
if (!foundTable.Exists())
{
return(Releaseability.ExtractFilesMissing);
}
return(Releaseability.Undefined);
}
/// <summary>
/// This version has less arguments because it goes back to the database and queries the configuration and explores who the cohort is etc, it will result in more database
/// queries than the more explicit constructor
/// </summary>
/// <param name="configuration"></param>
/// <param name="datasetBundle"></param>
/// <param name="includeValidation"></param>
/// <param name="includeLookups"></param>
public ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableDatasetBundle datasetBundle, bool includeValidation = false, bool includeLookups = false) : this(configuration, datasetBundle.DataSet)
{
DataExportRepository = configuration.DataExportRepository;
//ExtractableCohort = ExtractableCohort.GetExtractableCohortByID((int) configuration.Cohort_ID);
ExtractableCohort = configuration.GetExtractableCohort();
DatasetBundle = datasetBundle;
ColumnsToExtract = new List <IColumn>(Configuration.GetAllExtractableColumnsFor(datasetBundle.DataSet));
Salt = new HICProjectSalt(Project);
Directory = new ExtractionDirectory(Project.ExtractionDirectory, configuration);
IncludeValidation = includeValidation;
TopX = -1;
}
public override int GetHashCode()
{
unchecked
{
int hashCode = ExtractionJobIdentifier.GetHashCode();
hashCode = (hashCode * 397) ^ ProjectNumber.GetHashCode();
hashCode = (hashCode * 397) ^ ExtractionDirectory.GetHashCode();
hashCode = (hashCode * 397) ^ JobSubmittedAt.GetHashCode();
hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode();
hashCode = (hashCode * 397) ^ IsNoFilterExtraction.GetHashCode();
return(hashCode);
}
}
public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
{
if (pipelineFailureExceptionIfAny != null && _releaseData != null)
{
try
{
int remnantsDeleted = 0;
foreach (ExtractionConfiguration configuration in _releaseData.ConfigurationsForRelease.Keys)
{
foreach (ReleaseLog remnant in configuration.ReleaseLog)
{
remnant.DeleteInDatabase();
remnantsDeleted++;
}
}
if (remnantsDeleted > 0)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Because release failed we are deleting ReleaseLogEntries, this resulted in " + remnantsDeleted + " deleted records, you will likely need to re-extract these datasets or retrieve them from the Release directory"));
}
}
catch (Exception e1)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Error occurred when trying to clean up remnant ReleaseLogEntries", e1));
}
}
if (pipelineFailureExceptionIfAny == null && _destinationFolder != null)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Data release succeded into:" + _destinationFolder));
//mark configuration as released
foreach (var config in _configurationReleased)
{
config.IsReleased = true;
config.SaveToDatabase();
}
if (ReleaseSettings.DeleteFilesOnSuccess)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Cleaning up..."));
ExtractionDirectory.CleanupExtractionDirectory(this, _project.ExtractionDirectory, _configurationReleased, listener);
}
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "All done!"));
}
}
public override int GetHashCode()
{
unchecked
{
int hashCode = ExtractionJobIdentifier.GetHashCode();
hashCode = (hashCode * 397) ^ Header.GetHashCode();
hashCode = (hashCode * 397) ^ ProjectNumber.GetHashCode();
hashCode = (hashCode * 397) ^ (int)JobStatus;
hashCode = (hashCode * 397) ^ ExtractionDirectory.GetHashCode();
hashCode = (hashCode * 397) ^ JobSubmittedAt.GetHashCode();
hashCode = (hashCode * 397) ^ KeyTag.GetHashCode();
hashCode = (hashCode * 397) ^ (int)KeyCount;
hashCode = (hashCode * 397) ^ (ExtractionModality != null ? ExtractionModality.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode();
hashCode = (hashCode * 397) ^ IsNoFilterExtraction.GetHashCode();
hashCode = (hashCode * 397) ^ (FailedJobInfoDoc != null ? FailedJobInfoDoc.GetHashCode() : 0);
return hashCode;
}
}
/// <summary>
/// Returns the extraction name (last part of projName/extractions/extractName)
/// </summary>
/// <returns></returns>
public string ExtractionName()
{
string[] split = ExtractionDirectory.Split('/', '\\');
return(split[^ 1]);
