public override ExitCodeType Run(GracefulCancellationToken loadCancellationToken, object payload = null)
{
// single job, so grab the first available LoadProgress and lock it
var loadProgresses = LoadProgressSelectionStrategy.GetAllLoadProgresses();
if (!loadProgresses.Any())
{
return(ExitCodeType.OperationNotRequired);
}
var loadProgress = loadProgresses.First();
// we don't need any other schedules the strategy may have given us
loadProgresses.Remove(loadProgress);
// Create the job factory
if (_scheduledJobFactory != null)
{
throw new Exception("Job factory should only be created once");
}
_scheduledJobFactory = new SingleScheduledJobFactory(loadProgress, JobDateGenerationStrategyFactory.Create(loadProgress, DataLoadEventListener), OverrideNumberOfDaysToLoad ?? loadProgress.DefaultNumberOfDaysToLoadEachTime, LoadMetadata, LogManager);
// If the job factory won't produce any jobs we can bail out here
if (!_scheduledJobFactory.HasJobs())
{
return(ExitCodeType.OperationNotRequired);
}
// Run the data load
JobProvider = _scheduledJobFactory;
return(base.Run(loadCancellationToken, payload));
}
protected ScheduledDataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator, ILoadMetadata loadMetadata, ICheckable preExecutionChecker, IDataLoadExecution loadExecution, JobDateGenerationStrategyFactory jobDateGenerationStrategyFactory, ILoadProgressSelectionStrategy loadProgressSelectionStrategy, int?overrideNumberOfDaysToLoad, ILogManager logManager, IDataLoadEventListener dataLoadEventListener, HICDatabaseConfiguration configuration)
: base(repositoryLocator, loadMetadata, preExecutionChecker, logManager, dataLoadEventListener, loadExecution, configuration)
{
JobDateGenerationStrategyFactory = jobDateGenerationStrategyFactory;
LoadProgressSelectionStrategy = loadProgressSelectionStrategy;
OverrideNumberOfDaysToLoad = overrideNumberOfDaysToLoad;
}
public void DateKnown_NoCache_SuggestSingleScheduleConsecutiveDateStrategy()
{
var lp = Mock.Of <ILoadProgress>(p => p.DataLoadProgress == new DateTime(2001, 01, 01));
var factory = new JobDateGenerationStrategyFactory(new SingleLoadProgressSelectionStrategy(lp));
Assert.AreEqual(typeof(SingleScheduleConsecutiveDateStrategy), factory.Create(lp, new ThrowImmediatelyDataLoadEventListener()).GetType());
}
public void NoDates()
{
var lp = Mock.Of <ILoadProgress>();
var factory = new JobDateGenerationStrategyFactory(new SingleLoadProgressSelectionStrategy(lp));
var ex = Assert.Throws <LoadOrCacheProgressUnclearException>(() => factory.Create(lp, new ThrowImmediatelyDataLoadEventListener()));
Assert.AreEqual("Don't know when to start the data load, both DataLoadProgress and OriginDate are null", ex.Message);
}
public void LoadProgress_JobFactory_NoDates()
{
var lp = WhenIHaveA <LoadProgress>();
lp.OriginDate = new DateTime(2001, 1, 1);
// We are fully up-to-date
lp.DataLoadProgress = DateTime.Now;
lp.Check(new ThrowImmediatelyCheckNotifier());
var stratFactory = new JobDateGenerationStrategyFactory(new AnyAvailableLoadProgressSelectionStrategy(lp.LoadMetadata));
var strat = stratFactory.Create(lp, new ThrowImmediatelyDataLoadEventListener());
var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.WorkDirectory), "LoadProgress_JobFactory_NoDates", true);
var lmd = lp.LoadMetadata;
lmd.LocationOfFlatFiles = dir.RootPath.FullName;
foreach (var cata in lmd.GetAllCatalogues())
{
cata.LoggingDataTask = "ff";
cata.SaveToDatabase();
}
lmd.SaveToDatabase();
var jobFactory = new SingleScheduledJobFactory(lp, strat, 999, lp.LoadMetadata, null);
var ex = Assert.Throws <Exception>(() => jobFactory.Create(RepositoryLocator, new ThrowImmediatelyDataLoadEventListener(), null));
Assert.AreEqual("DatesToRetrieve was empty for load 'MyLoad'. Possibly the load is already up to date?", ex.Message);
// We have 1 day to load (date is the last fully loaded date)
lp.DataLoadProgress = DateTime.Now.AddDays(-2);
lp.SaveToDatabase();
strat = stratFactory.Create(lp, new ThrowImmediatelyDataLoadEventListener());
jobFactory = new SingleScheduledJobFactory(lp, strat, 999, lp.LoadMetadata, null);
var job = jobFactory.Create(RepositoryLocator, new ThrowImmediatelyDataLoadEventListener(), null);
Assert.AreEqual(1, ((ScheduledDataLoadJob)job).DatesToRetrieve.Count);
}
public void up()
{
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataWriter));
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataInventor));
_lmd = new LoadMetadata(CatalogueRepository, "JobDateGenerationStrategyFactoryTestsIntegration");
_lp = new LoadProgress(CatalogueRepository, _lmd);
_lp.DataLoadProgress = new DateTime(2001, 1, 1);
_lp.SaveToDatabase();
_cp = new CacheProgress(CatalogueRepository, _lp);
_server = new DiscoveredServer(new SqlConnectionStringBuilder("server=localhost;initial catalog=fish"));
_factory = new JobDateGenerationStrategyFactory(new SingleLoadProgressSelectionStrategy(_lp));
}
public override ExitCodeType Run(GracefulCancellationToken loadCancellationToken, object payload = null)
{
// grab all the load schedules we can and lock them
var loadProgresses = LoadProgressSelectionStrategy.GetAllLoadProgresses();
if (!loadProgresses.Any())
{
return(ExitCodeType.OperationNotRequired);
}
// create job factory
var progresses = loadProgresses.ToDictionary(loadProgress => loadProgress, loadProgress => JobDateGenerationStrategyFactory.Create(loadProgress, DataLoadEventListener));
var jobProvider = new MultipleScheduleJobFactory(progresses, OverrideNumberOfDaysToLoad, LoadMetadata, LogManager);
// check if the factory will produce any jobs, if not we can stop here
if (!jobProvider.HasJobs())
{
return(ExitCodeType.OperationNotRequired);
}
// Run the data load process
JobProvider = jobProvider;
//Do a data load
ExitCodeType result;
while ((result = base.Run(loadCancellationToken, payload)) == ExitCodeType.Success) //stop if it said not required
{
//or if between executions the token is set
if (loadCancellationToken.IsAbortRequested)
{
return(ExitCodeType.Abort);
}
if (loadCancellationToken.IsCancellationRequested)
{
return(ExitCodeType.Success);
}
}
//should be Operation Not Required or Error since the token inside handles stopping
return(result);
}
public int Run(IRDMPPlatformRepositoryServiceLocator locator, IDataLoadEventListener listener, ICheckNotifier checkNotifier, GracefulCancellationToken token)
{
ILoadProgress loadProgress = locator.CatalogueRepository.GetObjectByID <LoadProgress>(_options.LoadProgress);
ILoadMetadata loadMetadata = locator.CatalogueRepository.GetObjectByID <LoadMetadata>(_options.LoadMetadata);
if (loadMetadata == null && loadProgress != null)
{
loadMetadata = loadProgress.LoadMetadata;
}
if (loadMetadata == null)
{
throw new ArgumentException("No Load Metadata specified");
}
if (loadProgress != null && loadProgress.LoadMetadata_ID != loadMetadata.ID)
{
throw new ArgumentException("The supplied LoadProgress does not belong to the supplied LoadMetadata load");
}
var databaseConfiguration = new HICDatabaseConfiguration(loadMetadata);
var flags = new HICLoadConfigurationFlags();
flags.ArchiveData = !_options.DoNotArchiveData;
flags.DoLoadToStaging = !_options.StopAfterRAW;
flags.DoMigrateFromStagingToLive = !_options.StopAfterSTAGING;
var checkable = new CheckEntireDataLoadProcess(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository.MEF);
switch (_options.Command)
{
case CommandLineActivity.run:
var loggingServer = loadMetadata.GetDistinctLoggingDatabase();
var logManager = new LogManager(loggingServer);
// Create the pipeline to pass into the DataLoadProcess object
var dataLoadFactory = new HICDataLoadFactory(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository, logManager);
IDataLoadExecution execution = dataLoadFactory.Create(listener);
IDataLoadProcess dataLoadProcess;
if (loadMetadata.LoadProgresses.Any())
{
//Then the load is designed to run X days of source data at a time
//Load Progress
ILoadProgressSelectionStrategy whichLoadProgress = loadProgress != null ? (ILoadProgressSelectionStrategy) new SingleLoadProgressSelectionStrategy(loadProgress) : new AnyAvailableLoadProgressSelectionStrategy(loadMetadata);
var jobDateFactory = new JobDateGenerationStrategyFactory(whichLoadProgress);
dataLoadProcess = _options.Iterative
? (IDataLoadProcess) new IterativeScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration) :
new SingleJobScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration);
}
else
{
//OnDemand
dataLoadProcess = new DataLoadProcess(locator, loadMetadata, checkable, logManager, listener, execution, databaseConfiguration);
}
var exitCode = dataLoadProcess.Run(token);
//return 0 for success or load not required otherwise return the exit code (which will be non zero so error)
return(exitCode == ExitCodeType.Success || exitCode == ExitCodeType.OperationNotRequired? 0: (int)exitCode);
case CommandLineActivity.check:
checkable.Check(checkNotifier);
return(0);
default:
throw new ArgumentOutOfRangeException();
}
}
