private void CreateAttacher(ITableInfo t, QueryBuilder qb, LoadMetadata lmd, LoadProgress loadProgressIfAny)
{
var pt = new ProcessTask(Activator.RepositoryLocator.CatalogueRepository, lmd, LoadStage.Mounting);
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.Name = "Read from " + t;
pt.Path = typeof(RemoteTableAttacher).FullName;
pt.SaveToDatabase();
pt.CreateArgumentsForClassIfNotExists <RemoteTableAttacher>();
pt.SetArgumentValue("RemoteServer", t.Server);
pt.SetArgumentValue("RemoteDatabaseName", t.GetDatabaseRuntimeName(LoadStage.PostLoad));
pt.SetArgumentValue("RemoteTableName", t.GetRuntimeName());
pt.SetArgumentValue("DatabaseType", DatabaseType.MicrosoftSQLServer);
pt.SetArgumentValue("RemoteSelectSQL", qb.SQL);
pt.SetArgumentValue("RAWTableName", t.GetRuntimeName(LoadBubble.Raw));
if (loadProgressIfAny != null)
{
pt.SetArgumentValue("Progress", loadProgressIfAny);
// pt.SetArgumentValue("ProgressUpdateStrategy", DataLoadProgressUpdateStrategy.UseMaxRequestedDay);
pt.SetArgumentValue("LoadNotRequiredIfNoRowsRead", true);
}
/*
*
* public DataLoadProgressUpdateInfo { get; set; }
*/
}
public override void Execute()
{
if (_type == null)
{
if (BasicActivator.SelectType("Process Type", GetProcessTaskTypes(), out Type chosen))
{
SetType(chosen);
}
else
{
return;
}
}
ProcessTask newTask = new ProcessTask(BasicActivator.RepositoryLocator.CatalogueRepository, _loadMetadata, _loadStage);
newTask.Path = _type.FullName;
newTask.ProcessTaskType = _processTaskType;
newTask.Name = _type.Name;
newTask.SaveToDatabase();
newTask.CreateArgumentsForClassIfNotExists(_type);
Publish(_loadMetadata);
Activate(newTask);
}
private void btnBrowse_Click(object sender, EventArgs e)
{
OpenFileDialog ofd = new OpenFileDialog();
ofd.Filter = "Sql Files|*.sql";
ofd.CheckFileExists = true;
string oldFileName = null;
//open the browse dialog at the location of the currently specified file
if (!string.IsNullOrWhiteSpace(_processTask.Path))
{
var fi = new FileInfo(_processTask.Path);
oldFileName = fi.Name;
if (fi.Exists && fi.Directory != null)
{
ofd.InitialDirectory = fi.Directory.FullName;
}
}
if (ofd.ShowDialog() == DialogResult.OK && !string.IsNullOrWhiteSpace(ofd.FileName))
{
//replace the old name with the new name for example if user specified task name is 'Run bob.sql to rename all 'Roberts' to 'Bob' then the user selects a different file e.g. "truncateAllTables.sql" then the new name becomes Run truncateAllTables.sql to rename all 'Roberts' to 'Bob'
if (oldFileName != null)
{
_processTask.Name = _processTask.Name.Replace(oldFileName, Path.GetFileName(ofd.FileName));
}
_processTask.Path = ofd.FileName;
_processTask.SaveToDatabase();
Activator.RefreshBus.Publish(this, new RefreshObjectEventArgs(_processTask));
LoadFile();
}
}
private void btnBrowse_Click(object sender, EventArgs e)
{
var dialog = new OpenFileDialog();
dialog.Filter = "Executables|*.exe";
dialog.CheckFileExists = true;
//open the browse dialog at the location of the currently specified file
if (!string.IsNullOrWhiteSpace(_processTask.Path))
{
var fi = new FileInfo(_processTask.Path);
if (fi.Exists && fi.Directory != null)
{
dialog.InitialDirectory = fi.Directory.FullName;
}
}
if (dialog.ShowDialog() == DialogResult.OK)
{
_processTask.Path = dialog.FileName;
_processTask.SaveToDatabase();
Activator.RefreshBus.Publish(this, new RefreshObjectEventArgs(_processTask));
SetupForFile();
}
}
public void CacheProvider_NoCacheProgress()
{
var pt1 = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt1.Path = typeof(BasicCacheDataProvider).FullName;
pt1.ProcessTaskType = ProcessTaskType.DataProvider;
pt1.Name = "Cache1";
pt1.SaveToDatabase();
var projDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true);
_lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
_lmd.SaveToDatabase();
var pipeAssembler = new TestDataPipelineAssembler("CacheProvider_Normal", CatalogueRepository);
pipeAssembler.ConfigureCacheProgressToUseThePipeline(_cp);
try
{
var ex = Assert.Throws <InvalidOperationException>(() => _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener()));
Assert.AreEqual("Caching has not begun for this CacheProgress (" + _cp.ID + "), so there is nothing to load and this strategy should not be used.", ex.Message);
}
finally
{
_cp.Pipeline_ID = null;
pipeAssembler.Destroy();
projDir.RootPath.Delete(true);
}
}
public void CacheProvider_NoPipeline()
{
var pt1 = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt1.Path = typeof(TestCachedFileRetriever).FullName;
pt1.ProcessTaskType = ProcessTaskType.DataProvider;
pt1.Name = "Cache1";
pt1.SaveToDatabase();
_cp.CacheFillProgress = new DateTime(1999, 1, 1);
_cp.Name = "MyTestCp";
_cp.SaveToDatabase();
pt1.CreateArgumentsForClassIfNotExists <TestCachedFileRetriever>();
var projDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true);
_lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
_lmd.SaveToDatabase();
try
{
var ex = Assert.Throws <Exception>(() => _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener()));
Assert.AreEqual("CacheProgress MyTestCp does not have a Pipeline configured on it", ex.Message);
}
finally
{
projDir.RootPath.Delete(true);
}
}
/// <summary>
/// Creates a <see cref="AnySeparatorFileAttacher"/> parcelled into a <see cref="ProcessTask"/> that reads CSV files in ForLoading
/// in the mounting stage of the load
/// </summary>
/// <param name="lmd">The load to create the <see cref="ProcessTask"/> in</param>
/// <param name="pattern">File pattern to load e.g. *.csv</param>
/// <param name="ti">The table to load (must be part of the <paramref name="lmd"/></param>
/// <param name="separator">The separator of the files e.g. ','</param>
/// <param name="ignoreColumns">Columns to ignore in the load</param>
/// <returns></returns>
protected ProcessTask CreateFlatFileAttacher(LoadMetadata lmd, string pattern, ITableInfo ti, string separator = ",", string ignoreColumns = "hic_dataLoadRunID")
{
var csvProcessTask = new ProcessTask(CatalogueRepository, lmd, LoadStage.Mounting);
var args = csvProcessTask.CreateArgumentsForClassIfNotExists <AnySeparatorFileAttacher>();
csvProcessTask.Path = typeof(AnySeparatorFileAttacher).FullName;
csvProcessTask.ProcessTaskType = ProcessTaskType.Attacher;
csvProcessTask.SaveToDatabase();
var filePattern = args.Single(a => a.Name == "FilePattern");
filePattern.SetValue(pattern);
filePattern.SaveToDatabase();
var tableToLoad = args.Single(a => a.Name == "TableToLoad");
tableToLoad.SetValue(ti);
tableToLoad.SaveToDatabase();
var separatorArg = args.Single(a => a.Name == "Separator");
separatorArg.SetValue(separator);
separatorArg.SaveToDatabase();
var ignoreDataLoadRunIDCol = args.Single(a => a.Name == "IgnoreColumns");
ignoreDataLoadRunIDCol.SetValue(ignoreColumns);
ignoreDataLoadRunIDCol.SaveToDatabase();
return(csvProcessTask);
}
public override void Execute()
{
base.Execute();
_sourceProcessTask.LoadStage = _targetStage.LoadStage;
_sourceProcessTask.SaveToDatabase();
Publish(_sourceProcessTask.LoadMetadata);
}
private void SaveAndShow(ProcessTask task)
{
task.Name = "Run '" + Path.GetFileName(task.Path) + "'";
task.SaveToDatabase();
Publish(_loadMetadata);
Activate(task);
}
public void EmptyFilePath(string path, ProcessTaskType typeThatRequiresFiles)
{
_task.ProcessTaskType = typeThatRequiresFiles;
_task.Path = path;
_task.SaveToDatabase();
var ex = Assert.Throws <Exception>(() => _checker.Check(new ThrowImmediatelyCheckNotifier()));
StringAssert.Contains("does not have a path specified", ex.Message);
}
private void CreateDilutionMutilation(KeyValuePair <PreLoadDiscardedColumn, IDilutionOperation> dilutionOp, LoadMetadata lmd)
{
var pt = new ProcessTask(Activator.RepositoryLocator.CatalogueRepository, lmd, LoadStage.AdjustStaging);
pt.CreateArgumentsForClassIfNotExists <Dilution>();
pt.ProcessTaskType = ProcessTaskType.MutilateDataTable;
pt.Name = "Dilute " + dilutionOp.Key.GetRuntimeName();
pt.Path = typeof(Dilution).FullName;
pt.SaveToDatabase();
pt.SetArgumentValue("ColumnToDilute", dilutionOp.Key);
pt.SetArgumentValue("Operation", dilutionOp.Value.GetType());
}
public void CacheProvider_NonCachingOne()
{
var pt = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt.Path = typeof(DoNothingDataProvider).FullName;
pt.ProcessTaskType = ProcessTaskType.DataProvider;
pt.Name = "DoNothing";
pt.SaveToDatabase();
var ex = Assert.Throws <CacheDataProviderFindingException>(() => _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener()));
Assert.IsTrue(ex.Message.StartsWith("LoadMetadata JobDateGenerationStrategyFactoryTestsIntegration has some DataProviders tasks but none of them wrap classes that implement ICachedDataProvider"));
}
public override void Execute()
{
base.Execute();
var destinationOrder = 0;
var lmd = _targetProcessTask.LoadMetadata;
if (_insertOption == InsertOption.InsertAbove)
{
destinationOrder = _targetProcessTask.Order - 1;
foreach (var pt in lmd.ProcessTasks)
{
//don't change the current one again
if (pt.Equals(_sourceProcessTask))
{
continue;
}
if (pt.Order <= destinationOrder)
{
pt.Order--;
pt.SaveToDatabase();
}
}
}
else
{
destinationOrder = _targetProcessTask.Order + 1;
foreach (var pt in lmd.ProcessTasks)
{
//don't change the current one again
if (pt.Equals(_sourceProcessTask))
{
continue;
}
if (pt.Order >= destinationOrder)
{
pt.Order++;
pt.SaveToDatabase();
}
}
}
_sourceProcessTask.Order = destinationOrder;
_sourceProcessTask.SaveToDatabase();
Publish(lmd);
}
public override void Execute()
{
ProcessTask newTask = new ProcessTask(BasicActivator.RepositoryLocator.CatalogueRepository, _loadMetadata, _loadStage);
newTask.Path = _type.FullName;
newTask.ProcessTaskType = _processTaskType;
newTask.Name = _type.Name;
newTask.SaveToDatabase();
newTask.CreateArgumentsForClassIfNotExists(_type);
Publish(_loadMetadata);
Activate(newTask);
}
private void AddTypeIntoStage(Type type, ProcessTaskType taskType)
{
var lmd = _loadStageNode.LoadMetadata;
var stage = _loadStageNode.LoadStage;
ProcessTask newTask = new ProcessTask((ICatalogueRepository)lmd.Repository, lmd, stage);
newTask.Path = type.FullName;
newTask.ProcessTaskType = taskType;
newTask.Name = type.Name;
newTask.SaveToDatabase();
Publish(lmd);
Activate(newTask);
}
private void CreateCSVProcessTask(LoadMetadata lmd, TableInfo ti, string regex)
{
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.Mounting);
pt.Path = typeof(AnySeparatorFileAttacher).FullName;
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.Name = "Load " + ti.GetRuntimeName();
pt.SaveToDatabase();
pt.CreateArgumentsForClassIfNotExists <AnySeparatorFileAttacher>();
pt.SetArgumentValue("FilePattern", regex);
pt.SetArgumentValue("Separator", ",");
pt.SetArgumentValue("TableToLoad", ti);
pt.Check(new ThrowImmediatelyCheckNotifier());
}
public void GatherAndShare_LoadMetadata_WithReferenceProcessTaskArgument()
{
//create an object
LoadMetadata lmd1 = WhenIHaveA <LoadMetadata>();
//setup Reflection / MEF
SetupMEF();
RuntimeTaskFactory f = new RuntimeTaskFactory(Repository);
var stg = Mock.Of <IStageArgs>(x =>
x.LoadStage == LoadStage.Mounting &&
x.DbInfo == new DiscoveredServer(new SqlConnectionStringBuilder()).ExpectDatabase("d"));
//create a single process task for the load
var pt1 = new ProcessTask(Repository, lmd1, LoadStage.Mounting);
pt1.ProcessTaskType = ProcessTaskType.MutilateDataTable;
pt1.LoadStage = LoadStage.AdjustRaw;
pt1.Path = typeof(SafePrimaryKeyCollisionResolverMutilation).FullName;
pt1.SaveToDatabase();
//give it a reference to an (unshared) object (ColumnInfo)
pt1.CreateArgumentsForClassIfNotExists(typeof(SafePrimaryKeyCollisionResolverMutilation));
var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "ColumnToResolveOn");
pta.SetValue(WhenIHaveA <ColumnInfo>());
pta.SaveToDatabase();
//check that reflection can assemble the master ProcessTask
MutilateDataTablesRuntimeTask t = (MutilateDataTablesRuntimeTask)f.Create(pt1, stg);
Assert.IsNotNull(((SafePrimaryKeyCollisionResolverMutilation)t.MEFPluginClassInstance).ColumnToResolveOn);
//share to the second repository (which won't have that ColumnInfo)
var lmd2 = ShareToNewRepository(lmd1);
//create a new reflection factory for the new repo
RuntimeTaskFactory f2 = new RuntimeTaskFactory(lmd2.CatalogueRepository);
lmd2.CatalogueRepository.MEF = MEF;
//when we create the shared instance it should not have a valid value for ColumnInfo (since it wasn't - and shouldn't be shared)
MutilateDataTablesRuntimeTask t2 = (MutilateDataTablesRuntimeTask)f2.Create(lmd2.ProcessTasks.Single(), stg);
Assert.IsNull(((SafePrimaryKeyCollisionResolverMutilation)t2.MEFPluginClassInstance).ColumnToResolveOn);
}
public void CacheProvider_Normal()
{
var pt1 = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt1.Path = typeof(BasicCacheDataProvider).FullName;
pt1.ProcessTaskType = ProcessTaskType.DataProvider;
pt1.Name = "Cache1";
pt1.SaveToDatabase();
_cp.CacheFillProgress = new DateTime(2010, 1, 1);
_cp.SaveToDatabase();
var projDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true);
_lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
_lmd.SaveToDatabase();
var pipeAssembler = new TestDataPipelineAssembler("CacheProvider_Normal", CatalogueRepository);
pipeAssembler.ConfigureCacheProgressToUseThePipeline(_cp);
try
{
var strategy = _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener());
Assert.AreEqual(typeof(SingleScheduleCacheDateTrackingStrategy), strategy.GetType());
var dates = strategy.GetDates(10, false);
Assert.AreEqual(0, dates.Count); //zero dates to load because no files in cache
File.WriteAllText(Path.Combine(projDir.Cache.FullName, "2001-01-02.zip"), "bobbobbobyobyobyobbzzztproprietarybitztreamzippy");
File.WriteAllText(Path.Combine(projDir.Cache.FullName, "2001-01-03.zip"), "bobbobbobyobyobyobbzzztproprietarybitztreamzippy");
File.WriteAllText(Path.Combine(projDir.Cache.FullName, "2001-01-05.zip"), "bobbobbobyobyobyobbzzztproprietarybitztreamzippy");
strategy = _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener());
Assert.AreEqual(typeof(SingleScheduleCacheDateTrackingStrategy), strategy.GetType());
dates = strategy.GetDates(10, false);
Assert.AreEqual(3, dates.Count); //zero dates to load because no files in cache
}
finally
{
_cp.Pipeline_ID = null;
pipeAssembler.Destroy();
projDir.RootPath.Delete(true);
}
}
private void SetupLoadProcessTasks(ICatalogueRepository catalogueRepository)
{
var attacherTask = new ProcessTask(catalogueRepository, LoadMetadata, LoadStage.Mounting)
{
Name = "Attach CSV file",
Order = 1,
Path = "Rdmp.Core.DataLoad.Modules.Attachers.AnySeparatorFileAttacher",
ProcessTaskType = ProcessTaskType.Attacher
};
attacherTask.SaveToDatabase();
// Not assigned to a variable as they will be magically available through the repository
var processTaskArgs = new List <Tuple <string, string, Type> >
{
new Tuple <string, string, Type>("FilePattern", "1.csv", typeof(string)),
new Tuple <string, string, Type>("TableName", "TestData", typeof(string)),
new Tuple <string, string, Type>("ForceHeaders", null, typeof(string)),
new Tuple <string, string, Type>("IgnoreQuotes", null, typeof(bool)),
new Tuple <string, string, Type>("IgnoreBlankLines", null, typeof(bool)),
new Tuple <string, string, Type>("ForceHeadersReplacesFirstLineInFile", null, typeof(bool)),
new Tuple <string, string, Type>("SendLoadNotRequiredIfFileNotFound", "false", typeof(bool)),
new Tuple <string, string, Type>("Separator", ",", typeof(string)),
new Tuple <string, string, Type>("TableToLoad", null, typeof(TableInfo)),
new Tuple <string, string, Type>("BadDataHandlingStrategy", BadDataHandlingStrategy.ThrowException.ToString(), typeof(BadDataHandlingStrategy)),
new Tuple <string, string, Type>("ThrowOnEmptyFiles", "true", typeof(bool)),
new Tuple <string, string, Type>("AttemptToResolveNewLinesInRecords", "true", typeof(bool)),
new Tuple <string, string, Type>("MaximumErrorsToReport", "0", typeof(int)),
new Tuple <string, string, Type>("IgnoreColumns", null, typeof(string)),
new Tuple <string, string, Type>("IgnoreBadReads", "false", typeof(bool)),
new Tuple <string, string, Type>("AddFilenameColumnNamed", null, typeof(string)),
};
foreach (var tuple in processTaskArgs)
{
var pta = new ProcessTaskArgument(catalogueRepository, attacherTask)
{
Name = tuple.Item1,
Value = tuple.Item2
};
pta.SetType(tuple.Item3);
pta.SaveToDatabase();
}
}
public void CacheProvider_TwoCachingOnes()
{
var pt1 = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt1.Path = typeof(TestCachedFileRetriever).FullName;
pt1.ProcessTaskType = ProcessTaskType.DataProvider;
pt1.Name = "Cache1";
pt1.SaveToDatabase();
var pt2 = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt2.Path = typeof(TestCachedFileRetriever).FullName;
pt2.ProcessTaskType = ProcessTaskType.DataProvider;
pt2.Name = "Cache2";
pt2.SaveToDatabase();
var ex = Assert.Throws <CacheDataProviderFindingException>(() => _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener()));
Assert.AreEqual("LoadMetadata JobDateGenerationStrategyFactoryTestsIntegration has multiple cache DataProviders tasks (Cache1,Cache2), you are only allowed 1", ex.Message);
}
public void GatherAndShare_LoadMetadata_WithRealProcessTask()
{
//create an object
LoadMetadata lmd1 = WhenIHaveA <LoadMetadata>();
SetupMEF();
var pt1 = new ProcessTask(Repository, lmd1, LoadStage.Mounting);
pt1.ProcessTaskType = ProcessTaskType.Attacher;
pt1.LoadStage = LoadStage.Mounting;
pt1.Path = typeof(AnySeparatorFileAttacher).FullName;
pt1.SaveToDatabase();
pt1.CreateArgumentsForClassIfNotExists(typeof(AnySeparatorFileAttacher));
var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "Separator");
pta.SetValue(",");
pta.SaveToDatabase();
var lmd2 = ShareToNewRepository(lmd1);
//different repos so not identical
Assert.IsFalse(ReferenceEquals(lmd1, lmd2));
AssertAreEqual(lmd1, lmd2);
var pt2 = lmd2.ProcessTasks.Single();
Assert.IsFalse(ReferenceEquals(pt1, pt2));
AssertAreEqual(pt1, pt2);
AssertAreEqual(pt1.GetAllArguments(), pt2.GetAllArguments());
RuntimeTaskFactory f = new RuntimeTaskFactory(Repository);
var stg = Mock.Of <IStageArgs>(x => x.LoadStage == LoadStage.Mounting);
f.Create(pt1, stg);
}
public void TestConstructionFromProcessTaskUsingDatabase()
{
const string expectedPath = @"\\a\fake\path.exe";
var loadMetadata = new LoadMetadata(CatalogueRepository);
var processTask = new ProcessTask(CatalogueRepository, loadMetadata, LoadStage.Mounting)
{
Name = "Test process task",
Path = expectedPath
};
processTask.SaveToDatabase();
var argument = new ProcessTaskArgument(CatalogueRepository, processTask)
{
Name = "DatabaseName",
Value = @"Foo_STAGING"
};
argument.SaveToDatabase();
try
{
var args =
new RuntimeArgumentCollection(processTask.ProcessTaskArguments.Cast <IArgument>().ToArray(), null);
var runtimeTask = new ExecutableRuntimeTask(processTask, args);
Assert.AreEqual(expectedPath, runtimeTask.ExeFilepath);
Assert.AreEqual(1, runtimeTask.RuntimeArguments.GetAllArgumentsOfType <string>().Count());
var dictionaryOfStringArguments = runtimeTask.RuntimeArguments.GetAllArgumentsOfType <string>().ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
Assert.IsNotNull(dictionaryOfStringArguments["DatabaseName"]);
Assert.AreEqual("Foo_STAGING", dictionaryOfStringArguments["DatabaseName"]);
}
finally
{
loadMetadata.DeleteInDatabase();
}
}
public void RuntimeTaskFactoryTest(string className)
{
var lmd = new LoadMetadata(CatalogueRepository);
var task = new ProcessTask(CatalogueRepository, lmd, LoadStage.GetFiles);
var f = new RuntimeTaskFactory(CatalogueRepository);
task.Path = className;
task.ProcessTaskType = ProcessTaskType.DataProvider;
task.SaveToDatabase();
try
{
var ex = Assert.Throws <Exception>(() => f.Create(task, new StageArgs(LoadStage.AdjustRaw, GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer), Mock.Of <ILoadDirectory>())));
Assert.IsTrue(ex.InnerException.Message.Contains("marked with DemandsInitialization but no corresponding argument was provided in ArgumentCollection"));
}
finally
{
task.DeleteInDatabase();
lmd.DeleteInDatabase();
}
}
public void TestPayloadInjection()
{
BulkTestsData b = new BulkTestsData(CatalogueRepository, DiscoveredDatabaseICanCreateRandomTablesIn, 10);
b.SetupTestData();
b.ImportAsCatalogue();
var lmd = new LoadMetadata(CatalogueRepository, "Loading");
lmd.LocationOfFlatFiles = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true).RootPath.FullName;
lmd.SaveToDatabase();
CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestPayloadAttacher));
b.catalogue.LoadMetadata_ID = lmd.ID;
b.catalogue.LoggingDataTask = "TestPayloadInjection";
b.catalogue.SaveToDatabase();
var lm = new LogManager(new ServerDefaults(CatalogueRepository).GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID));
lm.CreateNewLoggingTaskIfNotExists("TestPayloadInjection");
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.Mounting);
pt.Path = typeof(TestPayloadAttacher).FullName;
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.SaveToDatabase();
var config = new HICDatabaseConfiguration(DiscoveredDatabaseICanCreateRandomTablesIn.Server);
var factory = new HICDataLoadFactory(lmd, config, new HICLoadConfigurationFlags(), CatalogueRepository, lm);
IDataLoadExecution execution = factory.Create(new ThrowImmediatelyDataLoadEventListener());
var proceedure = new DataLoadProcess(RepositoryLocator, lmd, null, lm, new ThrowImmediatelyDataLoadEventListener(), execution, config);
proceedure.Run(new GracefulCancellationToken(), payload);
Assert.IsTrue(PayloadTest.Success, "Expected IAttacher to detect Payload and set this property to true");
}
public void CloneProcessTask_ToSameLoadMetadataWithoutArguments()
{
LoadMetadata test = new LoadMetadata(CatalogueRepository);
ProcessTask processTask1 = new ProcessTask(CatalogueRepository, test, LoadStage.AdjustRaw)
{
Name = "Franky",
Order = 999
};
try
{
processTask1.SaveToDatabase();
var clone = processTask1.CloneToNewLoadMetadataStage(test, LoadStage.GetFiles);
Assert.AreNotSame(clone.ID, processTask1.ID);
Assert.IsFalse(clone.ID == processTask1.ID);
//get fresh copy out of database to ensure it is still there
ProcessTask orig = CatalogueRepository.GetObjectByID <ProcessTask>(processTask1.ID);
clone = CatalogueRepository.GetObjectByID <ProcessTask>(clone.ID);
Assert.IsFalse(orig.ID == clone.ID);
Assert.AreEqual(LoadStage.AdjustRaw, orig.LoadStage);
Assert.AreEqual(LoadStage.GetFiles, clone.LoadStage);
Assert.AreEqual(orig.Order, clone.Order);
Assert.AreEqual(orig.Path, clone.Path);
Assert.AreEqual(orig.ProcessTaskType, clone.ProcessTaskType);
Assert.AreEqual(orig.LoadMetadata_ID, clone.LoadMetadata_ID);
clone.DeleteInDatabase();
}
finally
{
processTask1.DeleteInDatabase();
test.DeleteInDatabase();
}
}
public void TestFindTables(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
var dt = new DataTable();
dt.Columns.Add("A");
dt.Columns.Add("B");
dt.Columns.Add("C");
// 'pk' 1 differs on col B AND col C
dt.Rows.Add(1, 2, 3);
dt.Rows.Add(1, 3, 2);
//novel (should not appear in diff table)
dt.Rows.Add(4, 1, 1);
//novel (should not appear in diff table)
dt.Rows.Add(5, 1, 1);
// 'pk' 2 differs on col C
dt.Rows.Add(2, 1, 1);
dt.Rows.Add(2, 1, 2);
//novel (should not appear in diff table)
dt.Rows.Add(6, 1, 1);
// 'pk' 3 differs on col B
dt.Rows.Add(3, 1, 1);
dt.Rows.Add(3, 2, 1);
db.CreateTable("mytbl_Isolation", dt);
var lmd = new LoadMetadata(CatalogueRepository, "ExampleLoad");
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw);
pt.ProcessTaskType = ProcessTaskType.MutilateDataTable;
pt.Path = typeof(PrimaryKeyCollisionIsolationMutilation).FullName;
pt.SaveToDatabase();
//make an isolation db that is the
var eds = new ExternalDatabaseServer(CatalogueRepository, "Isolation db", null);
eds.SetProperties(db);
var args = pt.CreateArgumentsForClassIfNotExists(typeof(PrimaryKeyCollisionIsolationMutilation));
var ti = new TableInfo(CatalogueRepository, "mytbl");
var ci = new ColumnInfo(CatalogueRepository, "A", "varchar(1)", ti);
ci.IsPrimaryKey = true;
ci.SaveToDatabase();
SetArg(args, "IsolationDatabase", eds);
SetArg(args, "TablesToIsolate", new [] { ti });
var reviewer = new IsolationReview(pt);
//no error since it is configured correctly
Assert.IsNull(reviewer.Error);
//tables should exist
var isolationTables = reviewer.GetIsolationTables();
Assert.IsTrue(isolationTables.Single().Value.Exists());
var diffDataTable = reviewer.GetDifferences(isolationTables.Single(), out List <IsolationDifference> diffs);
Assert.AreEqual(6, diffDataTable.Rows.Count);
Assert.AreEqual(6, diffs.Count);
}
protected override void SetUp()
{
base.SetUp();
Database = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
var rootFolder = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
var subdir = rootFolder.CreateSubdirectory("TestsRequiringADle");
LoadDirectory = LoadDirectory.CreateDirectoryStructure(rootFolder, subdir.FullName, true);
Clear(LoadDirectory);
LiveTable = CreateDataset <Demography>(Database, 500, 5000, new Random(190));
LiveTable.CreatePrimaryKey(new DiscoveredColumn[] {
LiveTable.DiscoverColumn("chi"),
LiveTable.DiscoverColumn("dtCreated"),
LiveTable.DiscoverColumn("hb_extract")
});
TestCatalogue = Import(LiveTable);
RowsBefore = 5000;
TestLoadMetadata = new LoadMetadata(CatalogueRepository, "Loading Test Catalogue");
TestLoadMetadata.LocationOfFlatFiles = LoadDirectory.RootPath.FullName;
TestLoadMetadata.SaveToDatabase();
//make the load load the table
TestCatalogue.LoadMetadata_ID = TestLoadMetadata.ID;
TestCatalogue.SaveToDatabase();
var csvProcessTask = new ProcessTask(CatalogueRepository, TestLoadMetadata, LoadStage.Mounting);
var args = csvProcessTask.CreateArgumentsForClassIfNotExists <AnySeparatorFileAttacher>();
csvProcessTask.Path = typeof(AnySeparatorFileAttacher).FullName;
csvProcessTask.ProcessTaskType = ProcessTaskType.Attacher;
csvProcessTask.SaveToDatabase();
var filePattern = args.Single(a => a.Name == "FilePattern");
filePattern.SetValue("*.csv");
filePattern.SaveToDatabase();
var tableToLoad = args.Single(a => a.Name == "TableToLoad");
tableToLoad.SetValue(TestCatalogue.GetTableInfoList(false).Single());
tableToLoad.SaveToDatabase();
var separator = args.Single(a => a.Name == "Separator");
separator.SetValue(",");
separator.SaveToDatabase();
var ignoreDataLoadRunIDCol = args.Single(a => a.Name == "IgnoreColumns");
ignoreDataLoadRunIDCol.SetValue("hic_dataLoadRunID");
ignoreDataLoadRunIDCol.SaveToDatabase();
//Get DleRunner to run pre load checks (includes trigger creation etc)
var runner = new DleRunner(new DleOptions()
{
LoadMetadata = TestLoadMetadata.ID, Command = CommandLineActivity.check
});
runner.Run(RepositoryLocator, new ThrowImmediatelyDataLoadEventListener(), new AcceptAllCheckNotifier(), new GracefulCancellationToken());
}
public void RunEndToEndDLECacheTest()
{
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataWriter));
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataInventor));
int timeoutInMilliseconds = 120000;
var lmd = TestLoadMetadata;
LoadProgress lp = new LoadProgress(CatalogueRepository, lmd);
lp.DataLoadProgress = new DateTime(2001, 1, 1);
lp.DefaultNumberOfDaysToLoadEachTime = 10;
lp.SaveToDatabase();
var cp = new CacheProgress(CatalogueRepository, lp);
cp.CacheFillProgress = new DateTime(2001, 1, 11); //10 days available to load
cp.SaveToDatabase();
var assembler = new TestDataPipelineAssembler("RunEndToEndDLECacheTest pipe", CatalogueRepository);
assembler.ConfigureCacheProgressToUseThePipeline(cp);
//setup the cache process task
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.GetFiles);
pt.Path = typeof(BasicCacheDataProvider).FullName;
pt.ProcessTaskType = ProcessTaskType.DataProvider;
pt.SaveToDatabase();
pt.CreateArgumentsForClassIfNotExists <BasicCacheDataProvider>();
var attacher = lmd.ProcessTasks.Single(p => p.ProcessTaskType == ProcessTaskType.Attacher);
var patternArgument = (ProcessTaskArgument)attacher.GetAllArguments().Single(a => a.Name.Equals("FilePattern"));
patternArgument.SetValue("*.csv");
patternArgument.SaveToDatabase();
//take the forLoading file
var csvFile = CreateFileInForLoading("bob.csv", 10, new Random(5000));
//and move it to the cache and give it a date in the range we expect for the cached data
csvFile.MoveTo(Path.Combine(LoadDirectory.Cache.FullName, "2001-01-09.csv"));
RunDLE(timeoutInMilliseconds);
Assert.AreEqual(10, RowsNow - RowsBefore);
Assert.AreEqual(0, LoadDirectory.Cache.GetFiles().Count());
Assert.AreEqual(0, LoadDirectory.ForLoading.GetFiles().Count());
Assert.AreEqual(1, LoadDirectory.ForArchiving.GetFiles().Count());
var archiveFile = LoadDirectory.ForArchiving.GetFiles()[0];
Assert.AreEqual(".zip", archiveFile.Extension);
//load progress should be updated to the largest date in the cache (2001-01-09)
lp.RevertToDatabaseState();
Assert.AreEqual(lp.DataLoadProgress, new DateTime(2001, 01, 09));
cp.DeleteInDatabase();
lp.DeleteInDatabase();
assembler.Destroy();
}
public override void Execute()
{
if (DicomSourceType == null)
{
SetImpossible("You must specify a Type for DicomSourceType");
throw new ImpossibleCommandException(this, ReasonCommandImpossible);
}
base.Execute();
List <DiscoveredTable> tablesCreated = new List <DiscoveredTable>();
//Create with template?
if (Template != null)
{
foreach (ImageTableTemplate table in Template.Tables)
{
string tblName = GetNameWithPrefix(table.TableName);
var tbl = _databaseToCreateInto.ExpectTable(tblName);
var cmd = new ExecuteCommandCreateNewImagingDataset(_repositoryLocator, tbl, table);
cmd.Execute();
NewCataloguesCreated.Add(cmd.NewCatalogueCreated);
tablesCreated.Add(tbl);
}
}
else
{
throw new Exception("No Template provided");
}
//that's us done if we aren't creating a load
if (!CreateLoad)
{
return;
}
string loadName = GetNameWithPrefixInBracketsIfAny("SMI Image Loading");
NewLoadMetadata = new LoadMetadata(_catalogueRepository, loadName);
//tell all the catalogues that they are part of this load and where to log under the same task
foreach (Catalogue c in NewCataloguesCreated)
{
c.LoadMetadata_ID = NewLoadMetadata.ID;
c.LoggingDataTask = loadName;
c.LiveLoggingServer_ID = _loggingServer.ID;
c.SaveToDatabase();
}
//create the logging task
new Core.Logging.LogManager(_loggingServer).CreateNewLoggingTaskIfNotExists(loadName);
var projDir = LoadDirectory.CreateDirectoryStructure(_projectDirectory, "ImageLoading", true);
NewLoadMetadata.LocationOfFlatFiles = projDir.RootPath.FullName;
NewLoadMetadata.SaveToDatabase();
/////////////////////////////////////////////Attacher////////////////////////////
//Create a pipeline for reading from Dicom files and writing to any destination component (which must be fixed)
var pipe = new Pipeline(_catalogueRepository, GetNameWithPrefixInBracketsIfAny("Image Loading Pipe"));
DicomSourcePipelineComponent = new PipelineComponent(_catalogueRepository, pipe, DicomSourceType, 0, DicomSourceType.Name);
DicomSourcePipelineComponent.CreateArgumentsForClassIfNotExists(DicomSourceType);
pipe.SourcePipelineComponent_ID = DicomSourcePipelineComponent.ID;
pipe.SaveToDatabase();
//Create the load process task that uses the pipe to load RAW tables with data from the dicom files
var pt = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.Mounting);
pt.Name = "Auto Routing Attacher";
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.Path = PersistentRaw? typeof(AutoRoutingAttacherWithPersistentRaw).FullName: typeof(AutoRoutingAttacher).FullName;
pt.Order = 1;
pt.SaveToDatabase();
var args = PersistentRaw? pt.CreateArgumentsForClassIfNotExists <AutoRoutingAttacherWithPersistentRaw>() : pt.CreateArgumentsForClassIfNotExists <AutoRoutingAttacher>();
SetArgument(args, "LoadPipeline", pipe);
/////////////////////////////////////// Distinct tables on load /////////////////////////
var distincter = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.AdjustRaw);
var distincterArgs = distincter.CreateArgumentsForClassIfNotExists <Distincter>();
distincter.Name = "Distincter";
distincter.ProcessTaskType = ProcessTaskType.MutilateDataTable;
distincter.Path = typeof(Distincter).FullName;
distincter.Order = 2;
distincter.SaveToDatabase();
SetArgument(distincterArgs, "TableRegexPattern", ".*");
/////////////////////////////////////////////////////////////////////////////////////
if (CreateCoalescer)
{
var coalescer = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.AdjustRaw);
coalescer.Name = "Coalescer";
coalescer.ProcessTaskType = ProcessTaskType.MutilateDataTable;
coalescer.Path = typeof(Coalescer).FullName;
coalescer.Order = 3;
coalescer.SaveToDatabase();
StringBuilder regexPattern = new StringBuilder();
foreach (var tbl in tablesCreated)
{
if (!tbl.DiscoverColumns().Any(c => c.GetRuntimeName().Equals("SOPInstanceUID", StringComparison.CurrentCultureIgnoreCase)))
{
regexPattern.Append("(" + tbl.GetRuntimeName() + ")|");
}
}
var coalArgs = coalescer.CreateArgumentsForClassIfNotExists <Coalescer>();
SetArgument(coalArgs, "TableRegexPattern", regexPattern.ToString().TrimEnd('|'));
SetArgument(coalArgs, "CreateIndex", true);
}
////////////////////////////////Load Ender (if no rows in load) ////////////////////////////
var prematureLoadEnder = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.Mounting);
prematureLoadEnder.Name = "Premature Load Ender";
prematureLoadEnder.ProcessTaskType = ProcessTaskType.MutilateDataTable;
prematureLoadEnder.Path = typeof(PrematureLoadEnder).FullName;
prematureLoadEnder.Order = 4;
prematureLoadEnder.SaveToDatabase();
args = prematureLoadEnder.CreateArgumentsForClassIfNotExists <PrematureLoadEnder>();
SetArgument(args, "ExitCodeToReturnIfConditionMet", ExitCodeType.OperationNotRequired);
SetArgument(args, "ConditionsToTerminateUnder", PrematureLoadEndCondition.NoRecordsInAnyTablesInDatabase);
////////////////////////////////////////////////////////////////////////////////////////////////
var checker = new CheckEntireDataLoadProcess(NewLoadMetadata, new HICDatabaseConfiguration(NewLoadMetadata), new HICLoadConfigurationFlags(), _catalogueRepository.MEF);
checker.Check(new AcceptAllCheckNotifier());
}
public void CloneProcessTask_ToNewLoadMetadataWithArguments()
{
//setup parents
LoadMetadata parent1 = new LoadMetadata(CatalogueRepository);
LoadMetadata parent2 = new LoadMetadata(CatalogueRepository);
//make sure we didn't magically create the same ID somehow
Assert.AreNotEqual(parent1.ID, parent2.ID);
//setup things to clone in parent1
ProcessTask processTask1 = new ProcessTask(CatalogueRepository, parent1, LoadStage.AdjustRaw);
ProcessTaskArgument arg = new ProcessTaskArgument(CatalogueRepository, processTask1);
arg.Name = "TestArg";
arg.SetType(typeof(System.String));
arg.SetValue("TestValue");
arg.SaveToDatabase();
processTask1.Name = "Franky";
processTask1.Order = 999;
processTask1.SaveToDatabase();
try
{
//clone to parent 2
var clone = processTask1.CloneToNewLoadMetadataStage(parent2, LoadStage.GetFiles);
Assert.AreNotSame(clone.ID, processTask1.ID);
Assert.IsFalse(clone.ID == processTask1.ID);
//////////////////////////////////////////////////////////////////CHECK CLONAGE OF PROCESS TASK ////////////////////////////////////////////////////////////
//get fresh copy out of database to ensure it is still there
ProcessTask orig = CatalogueRepository.GetObjectByID <ProcessTask>(processTask1.ID);
clone = CatalogueRepository.GetObjectByID <ProcessTask>(clone.ID);
//ids must have changed
Assert.IsFalse(orig.ID == clone.ID);
//load stages must be correct per what we requested
Assert.AreEqual(LoadStage.AdjustRaw, orig.LoadStage);
Assert.AreEqual(LoadStage.GetFiles, clone.LoadStage);
//all regular values must have been cloned successfully
Assert.AreEqual(orig.Order, clone.Order);
Assert.AreEqual(orig.Path, clone.Path);
Assert.AreEqual(orig.ProcessTaskType, clone.ProcessTaskType);
Assert.AreEqual(parent1.ID, orig.LoadMetadata_ID);
Assert.AreEqual(parent2.ID, clone.LoadMetadata_ID);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////CHECK CLONAGE OF ARGUMENTS ////////////////////////////////////////////////////////////
ProcessTaskArgument clonearg = clone.ProcessTaskArguments.SingleOrDefault();
Assert.NotNull(clonearg);
Assert.AreNotEqual(clonearg.ID, arg.ID);
Assert.AreEqual(clonearg.GetType(), arg.GetType());
Assert.AreEqual(clonearg.Name, arg.Name);
Assert.AreEqual(clonearg.Value, arg.Value);
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
clone.DeleteInDatabase();
}
finally
{
processTask1.DeleteInDatabase();
parent1.DeleteInDatabase();
parent2.DeleteInDatabase();
}
}
