protected override void SetUp()
{
base.SetUp();
var workingDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
parentDir = workingDir.CreateSubdirectory("FlatFileAttacherTests");
DirectoryInfo toCleanup = parentDir.GetDirectories().SingleOrDefault(d => d.Name.Equals("Test_CSV_Attachment"));
if (toCleanup != null)
{
toCleanup.Delete(true);
}
LoadDirectory = LoadDirectory.CreateDirectoryStructure(parentDir, "Test_CSV_Attachment");
// create a separate builder for setting an initial catalog on (need to figure out how best to stop child classes changing ServerICan... as this then causes TearDown to fail)
_database = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
using (var con = _database.Server.GetConnection())
{
con.Open();
var cmdCreateTable = _database.Server.GetCommand("CREATE Table " + _database.GetRuntimeName() + "..Bob([name] [varchar](500),[name2] [varchar](500))", con);
cmdCreateTable.ExecuteNonQuery();
}
_table = _database.ExpectTable("Bob");
}
public void MEFCompatibleType_NoArgs()
{
var projDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "DelMeProjDir", true);
try
{
_lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
_task.ProcessTaskType = ProcessTaskType.Attacher;
_task.LoadStage = LoadStage.Mounting;
_task.Path = typeof(AnySeparatorFileAttacher).FullName;
_task.SaveToDatabase();
var ex = Assert.Throws <ArgumentException>(() => _checker.Check(new ThrowImmediatelyCheckNotifier()
{
ThrowOnWarning = true
}));
Assert.AreEqual(@"Class AnySeparatorFileAttacher has a Mandatory property 'Separator' marked with DemandsInitialization but no corresponding argument was provided in ArgumentCollection", ex.Message);
}
finally
{
//delete everything for real
projDir.RootPath.Delete(true);
}
}
public void CreateArchiveWithNoFiles_ShouldThrow()
{
var directoryHelper = new TestDirectoryHelper(GetType());
directoryHelper.SetUp();
var testDir = directoryHelper.Directory.CreateSubdirectory("CreateArchiveWithNoFiles_ShouldThrow");
var archiveFiles = new ArchiveFiles(new HICLoadConfigurationFlags());
var loadDirectory = LoadDirectory.CreateDirectoryStructure(testDir, "dataset");
var job = Mock.Of <IDataLoadJob>(j => j.DataLoadInfo == Mock.Of <IDataLoadInfo>());
job.LoadDirectory = loadDirectory;
try
{
archiveFiles.Run(job, new GracefulCancellationToken());
foreach (FileInfo fileInfo in loadDirectory.ForArchiving.GetFiles("*.zip"))
{
Console.WriteLine("About to throw SetUp because of zip file:" + fileInfo.FullName);
}
Assert.IsFalse(loadDirectory.ForArchiving.GetFiles("*.zip").Any(), "There should not be any zip files in the archive directory!");
}
finally
{
directoryHelper.TearDown();
}
}
public void TestFactory()
{
var workingDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
var testDir = workingDir.CreateSubdirectory("MDFAttacherTests_TestFactory");
var loadDirectory = LoadDirectory.CreateDirectoryStructure(testDir, "TestFactory", true);
try
{
var attacher = CatalogueRepository.MEF.CreateA <IAttacher>(typeof(MDFAttacher).FullName);
attacher.Initialize(loadDirectory, GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer));
Assert.IsNotNull(attacher);
Assert.IsInstanceOf <MDFAttacher>(attacher);
}
finally
{
try
{
testDir.Delete(true);
}
catch (IOException e)
{
Console.WriteLine(e);
}
}
}
public void CacheProvider_NoPipeline()
{
var pt1 = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt1.Path = typeof(TestCachedFileRetriever).FullName;
pt1.ProcessTaskType = ProcessTaskType.DataProvider;
pt1.Name = "Cache1";
pt1.SaveToDatabase();
_cp.CacheFillProgress = new DateTime(1999, 1, 1);
_cp.Name = "MyTestCp";
_cp.SaveToDatabase();
pt1.CreateArgumentsForClassIfNotExists <TestCachedFileRetriever>();
var projDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true);
_lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
_lmd.SaveToDatabase();
try
{
var ex = Assert.Throws <Exception>(() => _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener()));
Assert.AreEqual("CacheProgress MyTestCp does not have a Pipeline configured on it", ex.Message);
}
finally
{
projDir.RootPath.Delete(true);
}
}
public void CacheProvider_NoCacheProgress()
{
var pt1 = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt1.Path = typeof(BasicCacheDataProvider).FullName;
pt1.ProcessTaskType = ProcessTaskType.DataProvider;
pt1.Name = "Cache1";
pt1.SaveToDatabase();
var projDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true);
_lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
_lmd.SaveToDatabase();
var pipeAssembler = new TestDataPipelineAssembler("CacheProvider_Normal", CatalogueRepository);
pipeAssembler.ConfigureCacheProgressToUseThePipeline(_cp);
try
{
var ex = Assert.Throws <InvalidOperationException>(() => _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener()));
Assert.AreEqual("Caching has not begun for this CacheProgress (" + _cp.ID + "), so there is nothing to load and this strategy should not be used.", ex.Message);
}
finally
{
_cp.Pipeline_ID = null;
pipeAssembler.Destroy();
projDir.RootPath.Delete(true);
}
}
private LoadDirectory CreateLoadDirectoryForTest(string directoryName)
{
var loadDirectory = LoadDirectory.CreateDirectoryStructure(_parentDir, directoryName, true);
_dirsToCleanUp.Push(loadDirectory.RootPath);
return(loadDirectory);
}
public void ExecuteSqlFileRuntimeTask_BasicScript(DatabaseType dbType)
{
var dt = new DataTable();
dt.Columns.Add("Lawl");
dt.Rows.Add(new object [] { 2 });
var db = GetCleanedServer(dbType, true);
var tbl = db.CreateTable("Fish", dt);
FileInfo f = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "Bob.sql"));
File.WriteAllText(f.FullName, @"UPDATE Fish Set Lawl = 1");
var pt = Mock.Of <IProcessTask>(x => x.Path == f.FullName);
var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true);
var task = new ExecuteSqlFileRuntimeTask(pt, new RuntimeArgumentCollection(new IArgument[0], new StageArgs(LoadStage.AdjustRaw, db, dir)));
task.Check(new ThrowImmediatelyCheckNotifier());
IDataLoadJob job = Mock.Of <IDataLoadJob>();
task.Run(job, new GracefulCancellationToken());
Assert.AreEqual(1, tbl.GetDataTable().Rows[0][0]);
tbl.Drop();
}
public void ConnectToServer()
{
var hicProjDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "MDFAttacherTest", true);
var db = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase("MyImaginaryDB_RAW");
Assert.IsFalse(db.Exists());
var mdf = new MDFAttacher();
mdf.Initialize(hicProjDir, db);
try
{
var memory = new ToMemoryCheckNotifier(new ThrowImmediatelyCheckNotifier());
mdf.Check(memory);
Assert.IsTrue(memory.Messages.Any(m => m.Message.Contains("Found server DATA folder") && m.Result == CheckResult.Success));
}
catch (Exception e)
{
if (!e.Message.Contains("Proposed server DATA folder (that we will copy mdf/ldf files to) was not found"))//this message is allowed too if the SQL server is remote and not localhost then it is quite likely that the DATA path is inaccessible from the unit test server
{
throw;
}
}
var memory2 = new ToMemoryCheckNotifier(new ThrowImmediatelyCheckNotifier());
mdf.OverrideMDFFileCopyDestination = TestContext.CurrentContext.WorkDirectory;
mdf.Check(memory2);
Assert.IsTrue(memory2.Messages.Any(m => Regex.IsMatch(m.Message, @"Found server DATA folder .*" + Regex.Escape(TestContext.CurrentContext.WorkDirectory)) && m.Result == CheckResult.Success));
hicProjDir.RootPath.Delete(true);
}
protected override void SetUp()
{
base.SetUp();
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataWriter));
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataInventor));
_lmd = new LoadMetadata(CatalogueRepository, "Ive got a lovely bunch o' coconuts");
_LoadDirectory = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), @"EndToEndCacheTest", true);
_lmd.LocationOfFlatFiles = _LoadDirectory.RootPath.FullName;
_lmd.SaveToDatabase();
Clear(_LoadDirectory);
_cata = new Catalogue(CatalogueRepository, "EndToEndCacheTest");
_cata.LoadMetadata_ID = _lmd.ID;
_cata.SaveToDatabase();
_lp = new LoadProgress(CatalogueRepository, _lmd);
_cp = new CacheProgress(CatalogueRepository, _lp);
_lp.OriginDate = new DateTime(2001, 1, 1);
_lp.SaveToDatabase();
_testPipeline = new TestDataPipelineAssembler("EndToEndCacheTestPipeline" + Guid.NewGuid(), CatalogueRepository);
_testPipeline.ConfigureCacheProgressToUseThePipeline(_cp);
_cp.CacheFillProgress = DateTime.Now.AddDays(-NumDaysToCache);
_cp.SaveToDatabase();
_cp.SaveToDatabase();
}
public void CreateTask()
{
_lmd = new LoadMetadata(CatalogueRepository);
_dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "ProcessTaskCheckingTests"));
_dir.Create();
var hicdir = LoadDirectory.CreateDirectoryStructure(_dir, "ProjDir", true);
_lmd.LocationOfFlatFiles = hicdir.RootPath.FullName;
_lmd.SaveToDatabase();
Catalogue c = new Catalogue(CatalogueRepository, "c");
CatalogueItem ci = new CatalogueItem(CatalogueRepository, c, "ci");
TableInfo t = new TableInfo(CatalogueRepository, "t");
t.Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name;
t.Database = "mydb";
t.SaveToDatabase();
ColumnInfo col = new ColumnInfo(CatalogueRepository, "col", "bit", t);
ci.SetColumnInfo(col);
c.LoadMetadata_ID = _lmd.ID;
c.SaveToDatabase();
_task = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
_checker = new ProcessTaskChecks(_lmd);
}
private LoadDirectory SetupLoadDirectory(LoadMetadata lmd)
{
var projectDirectory = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "MyLoadDir", true);
lmd.LocationOfFlatFiles = projectDirectory.RootPath.FullName;
lmd.SaveToDatabase();
return(projectDirectory);
}
public void ExecuteSqlRuntimeTask_InvalidID(DatabaseType dbType)
{
var dt = new DataTable();
dt.Columns.Add("Lawl");
dt.Rows.Add(new object[] { 2 });
var db = GetCleanedServer(dbType, true);
var tbl = db.CreateTable("Fish", dt);
TableInfo ti;
ColumnInfo[] cols;
Import(tbl, out ti, out cols);
string sql = @"UPDATE {T:0} Set {C:0} = 1";
IRuntimeTask task;
IProcessTask pt;
var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true);
var sqlArg = new IArgument[] { Mock.Of <IArgument>(x =>
x.Name == "Sql" &&
x.Value == sql &&
x.GetValueAsSystemType() == sql) };
var args = new RuntimeArgumentCollection(sqlArg, new StageArgs(LoadStage.AdjustRaw, db, dir));
pt = Mock.Of <IProcessTask>(x =>
x.Path == typeof(ExecuteSqlMutilation).FullName &&
x.GetAllArguments() == sqlArg
);
task = new MutilateDataTablesRuntimeTask(pt, args, CatalogueRepository.MEF);
task.Check(new ThrowImmediatelyCheckNotifier());
HICDatabaseConfiguration configuration = new HICDatabaseConfiguration(db.Server);
var job = new ThrowImmediatelyDataLoadJob();
job.RegularTablesToLoad = new List <ITableInfo> {
ti
};
job.LookupTablesToLoad = new List <ITableInfo>();
job.Configuration = configuration;
var ex = Assert.Throws <Exception>(() => task.Run(job, new GracefulCancellationToken()));
StringAssert.Contains("Mutilate failed", ex.Message);
StringAssert.Contains("Failed to find a TableInfo in the load with ID 0", ex.InnerException.Message);
task.LoadCompletedSoDispose(Core.DataLoad.ExitCodeType.Success, new ThrowImmediatelyDataLoadEventListener());
}
public void ExecuteSqlFileRuntimeTask_ValidID_CustomNamer(DatabaseType dbType)
{
var dt = new DataTable();
dt.Columns.Add("Lawl");
dt.Rows.Add(new object[] { 2 });
var db = GetCleanedServer(dbType, true);
var tbl = db.CreateTable("Fish", dt);
var tableName = "AAAAAAA";
TableInfo ti;
ColumnInfo[] cols;
Import(tbl, out ti, out cols);
FileInfo f = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "Bob.sql"));
File.WriteAllText(f.FullName, @"UPDATE {T:" + ti.ID + "} Set {C:" + cols[0].ID + "} = 1");
tbl.Rename(tableName);
//we renamed the table to simulate RAW, confirm TableInfo doesn't think it exists
Assert.IsFalse(ti.Discover(DataAccessContext.InternalDataProcessing).Exists());
var pt = Mock.Of <IProcessTask>(x => x.Path == f.FullName);
var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true);
var task = new ExecuteSqlFileRuntimeTask(pt, new RuntimeArgumentCollection(new IArgument[0], new StageArgs(LoadStage.AdjustRaw, db, dir)));
task.Check(new ThrowImmediatelyCheckNotifier());
//create a namer that tells the user
var namer = RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, tableName);
HICDatabaseConfiguration configuration = new HICDatabaseConfiguration(db.Server, namer);
IDataLoadJob job = Mock.Of <IDataLoadJob>(x =>
x.RegularTablesToLoad == new List <ITableInfo> {
ti
} &&
x.LookupTablesToLoad == new List <ITableInfo>() &&
x.Configuration == configuration);
task.Run(job, new GracefulCancellationToken());
Assert.AreEqual(1, tbl.GetDataTable().Rows[0][0]);
tbl.Drop();
}
public void LoadProgress_JobFactory_NoDates()
{
var lp = WhenIHaveA <LoadProgress>();
lp.OriginDate = new DateTime(2001, 1, 1);
// We are fully up-to-date
lp.DataLoadProgress = DateTime.Now;
lp.Check(new ThrowImmediatelyCheckNotifier());
var stratFactory = new JobDateGenerationStrategyFactory(new AnyAvailableLoadProgressSelectionStrategy(lp.LoadMetadata));
var strat = stratFactory.Create(lp, new ThrowImmediatelyDataLoadEventListener());
var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.WorkDirectory), "LoadProgress_JobFactory_NoDates", true);
var lmd = lp.LoadMetadata;
lmd.LocationOfFlatFiles = dir.RootPath.FullName;
foreach (var cata in lmd.GetAllCatalogues())
{
cata.LoggingDataTask = "ff";
cata.SaveToDatabase();
}
lmd.SaveToDatabase();
var jobFactory = new SingleScheduledJobFactory(lp, strat, 999, lp.LoadMetadata, null);
var ex = Assert.Throws <Exception>(() => jobFactory.Create(RepositoryLocator, new ThrowImmediatelyDataLoadEventListener(), null));
Assert.AreEqual("DatesToRetrieve was empty for load 'MyLoad'. Possibly the load is already up to date?", ex.Message);
// We have 1 day to load (date is the last fully loaded date)
lp.DataLoadProgress = DateTime.Now.AddDays(-2);
lp.SaveToDatabase();
strat = stratFactory.Create(lp, new ThrowImmediatelyDataLoadEventListener());
jobFactory = new SingleScheduledJobFactory(lp, strat, 999, lp.LoadMetadata, null);
var job = jobFactory.Create(RepositoryLocator, new ThrowImmediatelyDataLoadEventListener(), null);
Assert.AreEqual(1, ((ScheduledDataLoadJob)job).DatesToRetrieve.Count);
}
public void Test_MDFFile_AlreadyExists()
{
var workingDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
var data = workingDir.CreateSubdirectory("data");
var testDir = workingDir.CreateSubdirectory("MDFAttacherTests");
var loadDirectory = LoadDirectory.CreateDirectoryStructure(testDir, "TestNoMDFFileFoundException", true);
try
{
// create mdf and ldf files (in ForLoading
File.WriteAllText(Path.Combine(loadDirectory.ForLoading.FullName, "MyFile.mdf"), "fish");
File.WriteAllText(Path.Combine(loadDirectory.ForLoading.FullName, "MyFile_log.ldf"), "fish");
//create an already existing file in the 'data' directory (immitates the copy to location)
File.WriteAllText(Path.Combine(data.FullName, "MyFile.mdf"), "fish");
var attacher = new MDFAttacher();
attacher.OverrideMDFFileCopyDestination = data.FullName;
attacher.Initialize(loadDirectory, GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer));
//should be a warning since overwritting is default behaviour
var ex = Assert.Throws <Exception>(() =>
attacher.Attach(
new ThrowImmediatelyDataLoadJob(new ThrowImmediatelyDataLoadEventListener()
{
ThrowOnWarning = true
})
, new GracefulCancellationToken())
);
StringAssert.Contains("mdf already exists", ex.Message);
}
finally
{
try
{
testDir.Delete(true);
}
catch (IOException e)
{
Console.WriteLine(e);
}
}
}
public void MEFCompatibleType_Passes()
{
var projDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "DelMeProjDir", true);
try
{
_lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
_task.ProcessTaskType = ProcessTaskType.Attacher;
_task.LoadStage = LoadStage.Mounting;
_task.Path = typeof(AnySeparatorFileAttacher).FullName;
_task.SaveToDatabase();
//create the arguments
var args = ProcessTaskArgument.CreateArgumentsForClassIfNotExists <AnySeparatorFileAttacher>(_task);
var tblName = (ProcessTaskArgument)args.Single(a => a.Name.Equals("TableName"));
tblName.Value = "MyExcitingTable";
tblName.SaveToDatabase();
var filePattern = (ProcessTaskArgument)args.Single(a => a.Name.Equals("FilePattern"));
filePattern.Value = "*.csv";
filePattern.SaveToDatabase();
var separator = (ProcessTaskArgument)args.Single(a => a.Name.Equals("Separator"));
separator.Value = ",";
separator.SaveToDatabase();
var results = new ToMemoryCheckNotifier();
_checker.Check(results);
foreach (var msg in results.Messages)
{
Console.WriteLine("(" + msg.Result + ")" + msg.Message);
if (msg.Ex != null)
{
Console.WriteLine(ExceptionHelper.ExceptionToListOfInnerMessages(msg.Ex));
}
}
Assert.AreEqual(CheckResult.Success, results.GetWorst());
}
finally
{
//delete everything for real
projDir.RootPath.Delete(true);
}
}
public ExecuteCommandPacsFetch(IBasicActivateItems activator, string start, string end, string remoteAeUri, int remotePort, string remoteAeTitle, string localAeUri, int localPort, string localAeTitle, string outDir, int maxRetries) : base(activator)
{
var startDate = DateTime.Parse(start);
var endDate = DateTime.Parse(end);
// Make something that kinda looks like a valid DLE load
var memory = new MemoryCatalogueRepository();
var lmd = new LoadMetadata(memory);
var dir = Directory.CreateDirectory(outDir);
var results = LoadDirectory.CreateDirectoryStructure(dir, "out", true);
lmd.LocationOfFlatFiles = results.RootPath.FullName;
lmd.SaveToDatabase();
var lp = new LoadProgress(memory, lmd);
var cp = new CacheProgress(memory, lp);
//Create the source component only and a valid request range to fetch
_source = new PACSSource
{
RemoteAEUri = new Uri("http://" + remoteAeUri),
RemoteAEPort = remotePort,
RemoteAETitle = remoteAeTitle,
LocalAEUri = new Uri("http://" + localAeUri),
LocalAEPort = localPort,
LocalAETitle = localAeTitle,
TransferTimeOutInSeconds = 50000,
Modality = "ALL",
MaxRetries = maxRetries
};
//<- rly? its not gonna pass without an http!?
_request = new BackfillCacheFetchRequest(BasicActivator.RepositoryLocator.CatalogueRepository, startDate)
{
ChunkPeriod = endDate.Subtract(startDate), CacheProgress = cp
};
//Initialize it
_source.PreInitialize(BasicActivator.RepositoryLocator.CatalogueRepository, new ThrowImmediatelyDataLoadEventListener {
WriteToConsole = true
});
_source.PreInitialize(this, new ThrowImmediatelyDataLoadEventListener {
WriteToConsole = true
});
}
public void TestWithEcho()
{
var source = new ProcessBasedCacheSource();
if (IsLinux)
{
source.Command = "/bin/echo";
source.Args = "Hey Thomas go get %s and store in %d";
}
else
{
source.Command = "cmd.exe";
source.Args = "/c echo Hey Thomas go get %s and store in %d";
}
source.TimeFormat = "dd/MM/yy";
source.ThrowOnNonZeroExitCode = true;
// What dates to load
var cp = WhenIHaveA <CacheProgress>();
cp.CacheFillProgress = new DateTime(2001, 12, 24);
cp.SaveToDatabase();
// Where to put files
var lmd = cp.LoadProgress.LoadMetadata;
var dir = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory);
var loadDir = LoadDirectory.CreateDirectoryStructure(dir, "blah", true);
lmd.LocationOfFlatFiles = loadDir.RootPath.FullName;
lmd.SaveToDatabase();
source.PreInitialize(new CacheFetchRequestProvider(cp), new ThrowImmediatelyDataLoadEventListener());
source.PreInitialize(cp.CatalogueRepository, new ThrowImmediatelyDataLoadEventListener());
source.PreInitialize(new PermissionWindow(cp.CatalogueRepository), new ThrowImmediatelyDataLoadEventListener());
var toMem = new ToMemoryDataLoadEventListener(true);
var fork = new ForkDataLoadEventListener(toMem, new ThrowImmediatelyDataLoadEventListener()
{
WriteToConsole = true
});
source.GetChunk(fork, new GracefulCancellationToken());
Assert.Contains($"Hey Thomas go get 24/12/01 and store in {Path.Combine(loadDir.Cache.FullName,"ALL")}", toMem.GetAllMessagesByProgressEventType()[ProgressEventType.Information].Select(v => v.Message).ToArray());
}
public void CacheProvider_Normal()
{
var pt1 = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt1.Path = typeof(BasicCacheDataProvider).FullName;
pt1.ProcessTaskType = ProcessTaskType.DataProvider;
pt1.Name = "Cache1";
pt1.SaveToDatabase();
_cp.CacheFillProgress = new DateTime(2010, 1, 1);
_cp.SaveToDatabase();
var projDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true);
_lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
_lmd.SaveToDatabase();
var pipeAssembler = new TestDataPipelineAssembler("CacheProvider_Normal", CatalogueRepository);
pipeAssembler.ConfigureCacheProgressToUseThePipeline(_cp);
try
{
var strategy = _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener());
Assert.AreEqual(typeof(SingleScheduleCacheDateTrackingStrategy), strategy.GetType());
var dates = strategy.GetDates(10, false);
Assert.AreEqual(0, dates.Count); //zero dates to load because no files in cache
File.WriteAllText(Path.Combine(projDir.Cache.FullName, "2001-01-02.zip"), "bobbobbobyobyobyobbzzztproprietarybitztreamzippy");
File.WriteAllText(Path.Combine(projDir.Cache.FullName, "2001-01-03.zip"), "bobbobbobyobyobyobbzzztproprietarybitztreamzippy");
File.WriteAllText(Path.Combine(projDir.Cache.FullName, "2001-01-05.zip"), "bobbobbobyobyobyobbzzztproprietarybitztreamzippy");
strategy = _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener());
Assert.AreEqual(typeof(SingleScheduleCacheDateTrackingStrategy), strategy.GetType());
dates = strategy.GetDates(10, false);
Assert.AreEqual(3, dates.Count); //zero dates to load because no files in cache
}
finally
{
_cp.Pipeline_ID = null;
pipeAssembler.Destroy();
projDir.RootPath.Delete(true);
}
}
private void btnOk_Click(object sender, EventArgs e)
{
if (rbCreateNew.Checked)
{
try
{
var dir = new DirectoryInfo(tbCreateNew.Text);
if (!dir.Exists)
{
dir.Create();
}
Result = LoadDirectory.CreateDirectoryStructure(dir.Parent, dir.Name).RootPath.FullName;
DialogResult = DialogResult.OK;
this.Close();
}
catch (Exception exception)
{
ExceptionViewer.Show(exception);
}
}
if (rbUseExisting.Checked)
{
try
{
var dir = new LoadDirectory(tbUseExisting.Text);
Result = dir.RootPath.FullName;
DialogResult = DialogResult.OK;
this.Close();
}
catch (Exception exception)
{
if (Activator.YesNo($"Path is invalid, use anyway? ({exception.Message})", "Invalid Path"))
{
Result = tbUseExisting.Text;
DialogResult = DialogResult.OK;
this.Close();
}
}
}
}
public void ExecuteSqlFileRuntimeTask_InvalidID(DatabaseType dbType)
{
var dt = new DataTable();
dt.Columns.Add("Lawl");
dt.Rows.Add(new object[] { 2 });
var db = GetCleanedServer(dbType, true);
var tbl = db.CreateTable("Fish", dt);
TableInfo ti;
ColumnInfo[] cols;
Import(tbl, out ti, out cols);
FileInfo f = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "Bob.sql"));
File.WriteAllText(f.FullName, @"UPDATE {T:0} Set {C:0} = 1");
var pt = Mock.Of <IProcessTask>(x => x.Path == f.FullName);
var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true);
var task = new ExecuteSqlFileRuntimeTask(pt, new RuntimeArgumentCollection(new IArgument[0], new StageArgs(LoadStage.AdjustRaw, db, dir)));
task.Check(new ThrowImmediatelyCheckNotifier());
HICDatabaseConfiguration configuration = new HICDatabaseConfiguration(db.Server);
IDataLoadJob job = Mock.Of <IDataLoadJob>(x =>
x.RegularTablesToLoad == new List <ITableInfo> {
ti
} &&
x.LookupTablesToLoad == new List <ITableInfo>() &&
x.Configuration == configuration);
var ex = Assert.Throws <ExecuteSqlFileRuntimeTaskException>(() => task.Run(job, new GracefulCancellationToken()));
StringAssert.Contains("Failed to find a TableInfo in the load with ID 0", ex.Message);
task.LoadCompletedSoDispose(Core.DataLoad.ExitCodeType.Success, new ThrowImmediatelyDataLoadEventListener());
}
protected override void SetUp()
{
base.SetUp();
Database = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
var rootFolder = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
var subdir = rootFolder.CreateSubdirectory("TestsRequiringADle");
LoadDirectory = LoadDirectory.CreateDirectoryStructure(rootFolder, subdir.FullName, true);
Clear(LoadDirectory);
LiveTable = CreateDataset <Demography>(Database, 500, 5000, new Random(190));
LiveTable.CreatePrimaryKey(new DiscoveredColumn[] {
LiveTable.DiscoverColumn("chi"),
LiveTable.DiscoverColumn("dtCreated"),
LiveTable.DiscoverColumn("hb_extract")
});
TestCatalogue = Import(LiveTable);
RowsBefore = 5000;
TestLoadMetadata = new LoadMetadata(CatalogueRepository, "Loading Test Catalogue");
TestLoadMetadata.LocationOfFlatFiles = LoadDirectory.RootPath.FullName;
TestLoadMetadata.SaveToDatabase();
//make the load load the table
TestCatalogue.LoadMetadata_ID = TestLoadMetadata.ID;
TestCatalogue.SaveToDatabase();
CreateFlatFileAttacher(TestLoadMetadata, "*.csv", TestCatalogue.GetTableInfoList(false).Single(), ",");
//Get DleRunner to run pre load checks (includes trigger creation etc)
var runner = new DleRunner(new DleOptions()
{
LoadMetadata = TestLoadMetadata.ID, Command = CommandLineActivity.check
});
runner.Run(RepositoryLocator, new ThrowImmediatelyDataLoadEventListener(), new AcceptAllCheckNotifier(), new GracefulCancellationToken());
}
public void AttemptToLoadDataWithFilesInForLoading_AgreementBetweenForLoadingAndCache()
{
var tempDirPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
var tempDir = Directory.CreateDirectory(tempDirPath);
try
{
// File in cache is the same file as in ForLoading (20160101.zip)
var loadDirectory = LoadDirectory.CreateDirectoryStructure(tempDir, "CachedFileRetriever");
var cachedFilePath = Path.Combine(loadDirectory.Cache.FullName, "2016-01-01.zip");
File.WriteAllText(cachedFilePath, "");
File.WriteAllText(Path.Combine(loadDirectory.ForLoading.FullName, "2016-01-01.zip"), "");
// Set up retriever
var cacheLayout = new ZipCacheLayoutOnePerDay(loadDirectory.Cache, new NoSubdirectoriesCachePathResolver());
var retriever = new TestCachedFileRetriever()
{
ExtractFilesFromArchive = false,
LoadProgress = _lpMock,
Layout = cacheLayout
};
// Set up job
var job = CreateTestJob(loadDirectory);
job.DatesToRetrieve = new List <DateTime>
{
new DateTime(2016, 01, 01)
};
// Should complete successfully, the file in ForLoading matches the job specification
retriever.Fetch(job, new GracefulCancellationToken());
// And ForLoading should still have the file in it (i.e. it hasn't mysteriously disappeared)
Assert.IsTrue(File.Exists(Path.Combine(loadDirectory.ForLoading.FullName, "2016-01-01.zip")));
}
finally
{
tempDir.Delete(true);
}
}
public void AttemptToLoadDataWithoutFilesInForLoading()
{
var tempDirPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
var tempDir = Directory.CreateDirectory(tempDirPath);
try
{
// File in cache only, no files in ForLoading
var loadDirectory = LoadDirectory.CreateDirectoryStructure(tempDir, "CachedFileRetriever");
var cachedFilePath = Path.Combine(loadDirectory.Cache.FullName, "2016-01-01.zip");
File.WriteAllText(cachedFilePath, "");
// Set up retriever
var cacheLayout = new ZipCacheLayoutOnePerDay(loadDirectory.Cache, new NoSubdirectoriesCachePathResolver());
var retriever = new TestCachedFileRetriever()
{
ExtractFilesFromArchive = false,
LoadProgress = _lpMock,
Layout = cacheLayout
};
// Set up job
var job = CreateTestJob(loadDirectory);
job.DatesToRetrieve = new List <DateTime>
{
new DateTime(2016, 01, 01)
};
// Should complete successfully, there are no files in ForLoading to worry about
retriever.Fetch(job, new GracefulCancellationToken());
// And the retriever should have copied the cached archive file into ForLoading
Assert.IsTrue(File.Exists(Path.Combine(loadDirectory.ForLoading.FullName, "2016-01-01.zip")));
}
finally
{
tempDir.Delete(true);
}
}
public override void Execute()
{
base.Execute();
var d = Dir;
string newFolderName = null;
// if called with an explicit full dir then that is where we create load folders
// otherwise get them to pick something that exists and then name a new folder to
// create
if (d == null)
{
d = BasicActivator.SelectDirectory("Directory to create in");
if (d == null)
{
return;
}
if (!BasicActivator.TypeText("New Folder Name", "Name", 255, null, out newFolderName, false))
{
return;
}
}
var loadDir =
string.IsNullOrWhiteSpace(newFolderName) ?
LoadDirectory.CreateDirectoryStructure(d.Parent, d.Name, true) :
LoadDirectory.CreateDirectoryStructure(d, newFolderName, true);
// if we have a load then update the path to this location we just created
if (LoadMetadata != null)
{
LoadMetadata.LocationOfFlatFiles = loadDir.RootPath.FullName;
LoadMetadata.SaveToDatabase();
Publish(LoadMetadata);
}
}
public void AttemptToLoadDataWithFilesInForLoading_DisagreementBetweenCacheAndForLoading()
{
var tempDirPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
var tempDir = Directory.CreateDirectory(tempDirPath);
try
{
// Different file in ForLoading than exists in cache
var loadDirectory = LoadDirectory.CreateDirectoryStructure(tempDir, "CachedFileRetriever");
var cachedFilePath = Path.Combine(loadDirectory.Cache.FullName, "2016-01-02.zip");
File.WriteAllText(cachedFilePath, "");
File.WriteAllText(Path.Combine(loadDirectory.ForLoading.FullName, "2016-01-01.zip"), "");
// Set up retriever
var cacheLayout = new ZipCacheLayoutOnePerDay(loadDirectory.Cache, new NoSubdirectoriesCachePathResolver());
var retriever = new TestCachedFileRetriever()
{
ExtractFilesFromArchive = false,
LoadProgress = _lpMock,
Layout = cacheLayout
};
// Set up job
var job = CreateTestJob(loadDirectory);
job.DatesToRetrieve = new List <DateTime>
{
new DateTime(2016, 01, 02)
};
// Should fail after determining that the files in ForLoading do not match the job specification
var ex = Assert.Throws <InvalidOperationException>(() => retriever.Fetch(job, new GracefulCancellationToken()));
Assert.IsTrue(ex.Message.StartsWith("The files in ForLoading do not match what this job expects to be loading from the cache."), ex.Message + Environment.NewLine + Environment.NewLine + ex.StackTrace);
}
finally
{
tempDir.Delete(true);
}
}
private void btnOk_Click(object sender, EventArgs e)
{
if (rbCreateNew.Checked)
{
try
{
var dir = new DirectoryInfo(tbCreateNew.Text);
if (!dir.Exists)
{
dir.Create();
}
Result = LoadDirectory.CreateDirectoryStructure(dir.Parent, dir.Name);
DialogResult = DialogResult.OK;
this.Close();
}
catch (Exception exception)
{
ExceptionViewer.Show(exception);
}
}
if (rbUseExisting.Checked)
{
try
{
Result = new LoadDirectory(tbUseExisting.Text);
DialogResult = DialogResult.OK;
this.Close();
}
catch (Exception exception)
{
ExceptionViewer.Show(exception);
}
}
}
public void TestPayloadInjection()
{
BulkTestsData b = new BulkTestsData(CatalogueRepository, DiscoveredDatabaseICanCreateRandomTablesIn, 10);
b.SetupTestData();
b.ImportAsCatalogue();
var lmd = new LoadMetadata(CatalogueRepository, "Loading");
lmd.LocationOfFlatFiles = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true).RootPath.FullName;
lmd.SaveToDatabase();
CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestPayloadAttacher));
b.catalogue.LoadMetadata_ID = lmd.ID;
b.catalogue.LoggingDataTask = "TestPayloadInjection";
b.catalogue.SaveToDatabase();
var lm = new LogManager(new ServerDefaults(CatalogueRepository).GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID));
lm.CreateNewLoggingTaskIfNotExists("TestPayloadInjection");
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.Mounting);
pt.Path = typeof(TestPayloadAttacher).FullName;
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.SaveToDatabase();
var config = new HICDatabaseConfiguration(DiscoveredDatabaseICanCreateRandomTablesIn.Server);
var factory = new HICDataLoadFactory(lmd, config, new HICLoadConfigurationFlags(), CatalogueRepository, lm);
IDataLoadExecution execution = factory.Create(new ThrowImmediatelyDataLoadEventListener());
var proceedure = new DataLoadProcess(RepositoryLocator, lmd, null, lm, new ThrowImmediatelyDataLoadEventListener(), execution, config);
proceedure.Run(new GracefulCancellationToken(), payload);
Assert.IsTrue(PayloadTest.Success, "Expected IAttacher to detect Payload and set this property to true");
}
public void TestNoMDFFileFoundException()
{
var workingDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
var testDir = workingDir.CreateSubdirectory("MDFAttacherTests");
var loadDirectory = LoadDirectory.CreateDirectoryStructure(testDir, "TestNoMDFFileFoundException", true);
try
{
var attacher = new MDFAttacher();
attacher.Initialize(loadDirectory, GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer));
Assert.Throws <FileNotFoundException>(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken()));
}
finally
{
try
{
testDir.Delete(true);
}
catch (IOException e)
{
Console.WriteLine(e);
}
}
}