/// <summary>
/// Runs the DLE using a custom names for RAW/STAGING. Pass in the load to execute and the files/directories to process
/// in the batch.
/// </summary>
/// <param name="lmd"></param>
/// <param name="payload"></param>
/// <returns>The exit code of the data load after it completes</returns>
private ExitCodeType RunDLE(LoadMetadata lmd, object payload)
{
var catalogueRepository = (CatalogueRepository)lmd.Repository;
//ensures that RAW/STAGING always have unique names
_configuration = new HICDatabaseConfiguration(lmd, _namer);
_configuration.UpdateButDoNotDiff = new Regex("^MessageGuid");
var logManager = catalogueRepository.GetDefaultLogManager();
logManager.CreateNewLoggingTaskIfNotExists(lmd.GetDistinctLoggingTask());
// Create the pipeline to pass into the DataLoadProcess object
var dataLoadFactory = new HICDataLoadFactory(lmd, _configuration, new HICLoadConfigurationFlags(),
catalogueRepository, logManager);
var stagingCreator = _namer as ICreateAndDestroyStagingDuringLoads;
if (stagingCreator != null)
{
stagingCreator.CreateStaging(lmd.GetDistinctLiveDatabaseServer());
}
var listener = new NLogThrowerDataLoadEventListener(NLog.LogManager.GetCurrentClassLogger());
IDataLoadExecution execution = dataLoadFactory.Create(listener);
IExternalDatabaseServer raw = catalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.RAWDataLoadServer);
DiscoveredServer liveDb = lmd.GetDistinctLiveDatabaseServer();
//do we want to try to cut down the time it takes to do RAW=>STAGING by using INSERT INTO instead of running anonymisation/migration pipeline
if (_useInsertIntoForRawMigration)
{
//if it is on the same server swap out the migration engine for INSERT INTO
if (raw == null || (raw.Server != null && raw.Server.Equals(liveDb.Name) && raw.DatabaseType == liveDb.DatabaseType))
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "SWAPPING RAW=>STAGING migration strategy to INSERT INTO"));
SwapMigrateRAWToStagingComponent(execution.Components);
}
else
{
//Cannot use because different servers / DatabaseTypes.
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "CANNOT SWAP RAW=>STAGING migration strategy to INSERT INTO because RAW is on '" + raw.Server + "' (" + raw.DatabaseType + ") and STAGING is on '" + liveDb.Name + "' (" + liveDb.DatabaseType + ")"));
}
}
else
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Flag is false for SWAP RAW=>STAGING migration strategy to INSERT INTO So won't do it"));
}
var procedure = new DataLoadProcess(_repositoryLocator, lmd, null, logManager, listener, execution, _configuration);
ExitCodeType exitCode = procedure.Run(new GracefulCancellationToken(), payload);
return(exitCode);
}
public void TestPayloadInjection()
{
BulkTestsData b = new BulkTestsData(CatalogueRepository, DiscoveredDatabaseICanCreateRandomTablesIn, 10);
b.SetupTestData();
b.ImportAsCatalogue();
var lmd = new LoadMetadata(CatalogueRepository, "Loading");
lmd.LocationOfFlatFiles = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true).RootPath.FullName;
lmd.SaveToDatabase();
CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestPayloadAttacher));
b.catalogue.LoadMetadata_ID = lmd.ID;
b.catalogue.LoggingDataTask = "TestPayloadInjection";
b.catalogue.SaveToDatabase();
var lm = new LogManager(new ServerDefaults(CatalogueRepository).GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID));
lm.CreateNewLoggingTaskIfNotExists("TestPayloadInjection");
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.Mounting);
pt.Path = typeof(TestPayloadAttacher).FullName;
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.SaveToDatabase();
var config = new HICDatabaseConfiguration(DiscoveredDatabaseICanCreateRandomTablesIn.Server);
var factory = new HICDataLoadFactory(lmd, config, new HICLoadConfigurationFlags(), CatalogueRepository, lm);
IDataLoadExecution execution = factory.Create(new ThrowImmediatelyDataLoadEventListener());
var proceedure = new DataLoadProcess(RepositoryLocator, lmd, null, lm, new ThrowImmediatelyDataLoadEventListener(), execution, config);
proceedure.Run(new GracefulCancellationToken(), payload);
Assert.IsTrue(PayloadTest.Success, "Expected IAttacher to detect Payload and set this property to true");
}
public int Run(IRDMPPlatformRepositoryServiceLocator locator, IDataLoadEventListener listener, ICheckNotifier checkNotifier, GracefulCancellationToken token)
{
ILoadProgress loadProgress = locator.CatalogueRepository.GetObjectByID <LoadProgress>(_options.LoadProgress);
ILoadMetadata loadMetadata = locator.CatalogueRepository.GetObjectByID <LoadMetadata>(_options.LoadMetadata);
if (loadMetadata == null && loadProgress != null)
{
loadMetadata = loadProgress.LoadMetadata;
}
if (loadMetadata == null)
{
throw new ArgumentException("No Load Metadata specified");
}
if (loadProgress != null && loadProgress.LoadMetadata_ID != loadMetadata.ID)
{
throw new ArgumentException("The supplied LoadProgress does not belong to the supplied LoadMetadata load");
}
var databaseConfiguration = new HICDatabaseConfiguration(loadMetadata);
var flags = new HICLoadConfigurationFlags();
flags.ArchiveData = !_options.DoNotArchiveData;
flags.DoLoadToStaging = !_options.StopAfterRAW;
flags.DoMigrateFromStagingToLive = !_options.StopAfterSTAGING;
var checkable = new CheckEntireDataLoadProcess(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository.MEF);
switch (_options.Command)
{
case CommandLineActivity.run:
var loggingServer = loadMetadata.GetDistinctLoggingDatabase();
var logManager = new LogManager(loggingServer);
// Create the pipeline to pass into the DataLoadProcess object
var dataLoadFactory = new HICDataLoadFactory(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository, logManager);
IDataLoadExecution execution = dataLoadFactory.Create(listener);
IDataLoadProcess dataLoadProcess;
if (loadMetadata.LoadProgresses.Any())
{
//Then the load is designed to run X days of source data at a time
//Load Progress
ILoadProgressSelectionStrategy whichLoadProgress = loadProgress != null ? (ILoadProgressSelectionStrategy) new SingleLoadProgressSelectionStrategy(loadProgress) : new AnyAvailableLoadProgressSelectionStrategy(loadMetadata);
var jobDateFactory = new JobDateGenerationStrategyFactory(whichLoadProgress);
dataLoadProcess = _options.Iterative
? (IDataLoadProcess) new IterativeScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration) :
new SingleJobScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration);
}
else
{
//OnDemand
dataLoadProcess = new DataLoadProcess(locator, loadMetadata, checkable, logManager, listener, execution, databaseConfiguration);
}
var exitCode = dataLoadProcess.Run(token);
//return 0 for success or load not required otherwise return the exit code (which will be non zero so error)
return(exitCode == ExitCodeType.Success || exitCode == ExitCodeType.OperationNotRequired? 0: (int)exitCode);
case CommandLineActivity.check:
checkable.Check(checkNotifier);
return(0);
default:
throw new ArgumentOutOfRangeException();
}
}
