/// <summary>
/// Defines that the given <see cref="LoadProgress"/> is a DLE data load that is driven by reading data from a cache. The instance created can be used
/// to describe which pipeline should be run to fill that cache, the period that has been fetched from the remote endpoint so far etc.
/// </summary>
/// <param name="repository"></param>
/// <param name="loadProgress"></param>
public CacheProgress(ICatalogueRepository repository, ILoadProgress loadProgress)
{
repository.InsertAndHydrate(this, new Dictionary <string, object>
{
{ "LoadProgress_ID", loadProgress.ID },
{ "Name", "New CacheProgress " + Guid.NewGuid() }
});
}
public ICacheLayout CreateCacheLayout(ILoadProgress loadProgress, ILoadMetadata metadata)
{
AssertThatThereIsACacheDataProvider(metadata, metadata.ProcessTasks.Where(p=>!p.IsDisabled));
var cp = loadProgress.CacheProgress;
var factory = new CachingPipelineUseCase(cp);
var destination = factory.CreateDestinationOnly(new ThrowImmediatelyDataLoadEventListener());
return destination.CreateCacheLayout();
}
public LoadProgressSummaryReport(LoadProgress loadProgress)
{
_loadProgress = loadProgress;
_loadMetadata = _loadProgress.LoadMetadata;
_cacheProgress = _loadProgress.CacheProgress;
try
{
dqeRepository = new DQERepository(loadProgress.CatalogueRepository);
}
catch (NotSupportedException)
{
dqeRepository = null;
}
}
public SingleScheduleConsecutiveDateStrategy(ILoadProgress loadProgress)
{
if (loadProgress.DataLoadProgress == null)
{
if (loadProgress.OriginDate == null)
{
throw new LoadOrCacheProgressUnclearException("Don't know when to start the data load, both DataLoadProgress and OriginDate are null");
}
_lastAssignedLoadDate = loadProgress.OriginDate.Value;
}
else
{
_lastAssignedLoadDate = loadProgress.DataLoadProgress.Value;
}
}
public SingleScheduleCacheDateTrackingStrategy(ICacheLayout cacheLayout, ILoadProgress loadProgress, IDataLoadEventListener listener)
{
// no null check needed as the contract ensures that both DataLoadProgress and OriginDate can't simultaneously be null
var lastAssignedLoadDate = loadProgress.DataLoadProgress == null ? loadProgress.OriginDate.Value : loadProgress.DataLoadProgress.Value;
// This is all the dates in the cache, but we want to start from _lastAssignedLoadDate
// todo: must be efficient, revisit
_availableDates = cacheLayout.GetSortedDateQueue(listener);
while (_availableDates.Any() && _availableDates.Peek() <= lastAssignedLoadDate)
{
_availableDates.Dequeue();
}
_lastDateForLoading = CalculateLastLoadDate(loadProgress);
}
public IJobDateGenerationStrategy Create(ILoadProgress loadProgress, IDataLoadEventListener listener)
{
if (_typeToCreate == typeof(SingleScheduleConsecutiveDateStrategy))
{
return(new SingleScheduleConsecutiveDateStrategy(loadProgress));
}
var loadMetadata = loadProgress.LoadMetadata;
var factory = new CacheLayoutFactory();
if (_typeToCreate == typeof(SingleScheduleCacheDateTrackingStrategy))
{
return(new SingleScheduleCacheDateTrackingStrategy(factory.CreateCacheLayout(loadProgress, loadMetadata), loadProgress, listener));
}
throw new Exception("Factory has been configured to supply an unknown type");
}
/// <summary>
///
/// </summary>
/// <param name="loadProgress"></param>
/// <returns></returns>
/// <exception cref="InvalidOperationException">Caching has not been configured correctly or the caching process has not begun</exception>
public DateTime CalculateLastLoadDate(ILoadProgress loadProgress)
{
// Compute the last cache date from the CacheFillProgress date
// CacheFillProgress is the date up to which caching has been performed, and is therefore the date from which caching will next begin.
var cacheProgress = loadProgress.CacheProgress;
if (cacheProgress == null)
{
throw new InvalidOperationException("Could not retrieve the CacheProgress from LoadProgress " + loadProgress.ID + " (ensure caching is configured on this load before using this strategy)");
}
if (cacheProgress.CacheFillProgress == null)
{
throw new InvalidOperationException("Caching has not begun for this CacheProgress (" + cacheProgress.ID + "), so there is nothing to load and this strategy should not be used.");
}
// We don't want to load partially filled cache files, so use the CacheFileGranularity to calculate the latest file we can safely load
// CacheFileGranularity is a caching pipeline component argument, so need to get the runtime pipeline
var cachedFileRetriever = GetCacheDestinationPipelineComponent(cacheProgress);
var layout = cachedFileRetriever.CreateCacheLayout();
return(CalculateLastLoadDate(layout.CacheFileGranularity, cacheProgress.CacheFillProgress.Value));
}
public CachedFileRetrieverTests()
{
_cpMock = Mock.Of <ICacheProgress>();
_lpMock = Mock.Of <ILoadProgress>(l => l.CacheProgress == _cpMock);
}
public SingleScheduledJobFactory(ILoadProgress loadProgress, IJobDateGenerationStrategy jobDateGenerationStrategy, int overrideNumberOfDaysToLoad, ILoadMetadata loadMetadata, ILogManager logManager) : base(overrideNumberOfDaysToLoad, loadMetadata, logManager)
{
_loadProgress = loadProgress;
_jobDateGenerationStrategy = jobDateGenerationStrategy;
}
public int Run(IRDMPPlatformRepositoryServiceLocator locator, IDataLoadEventListener listener, ICheckNotifier checkNotifier, GracefulCancellationToken token)
{
ILoadProgress loadProgress = locator.CatalogueRepository.GetObjectByID <LoadProgress>(_options.LoadProgress);
ILoadMetadata loadMetadata = locator.CatalogueRepository.GetObjectByID <LoadMetadata>(_options.LoadMetadata);
if (loadMetadata == null && loadProgress != null)
{
loadMetadata = loadProgress.LoadMetadata;
}
if (loadMetadata == null)
{
throw new ArgumentException("No Load Metadata specified");
}
if (loadProgress != null && loadProgress.LoadMetadata_ID != loadMetadata.ID)
{
throw new ArgumentException("The supplied LoadProgress does not belong to the supplied LoadMetadata load");
}
var databaseConfiguration = new HICDatabaseConfiguration(loadMetadata);
var flags = new HICLoadConfigurationFlags();
flags.ArchiveData = !_options.DoNotArchiveData;
flags.DoLoadToStaging = !_options.StopAfterRAW;
flags.DoMigrateFromStagingToLive = !_options.StopAfterSTAGING;
var checkable = new CheckEntireDataLoadProcess(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository.MEF);
switch (_options.Command)
{
case CommandLineActivity.run:
var loggingServer = loadMetadata.GetDistinctLoggingDatabase();
var logManager = new LogManager(loggingServer);
// Create the pipeline to pass into the DataLoadProcess object
var dataLoadFactory = new HICDataLoadFactory(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository, logManager);
IDataLoadExecution execution = dataLoadFactory.Create(listener);
IDataLoadProcess dataLoadProcess;
if (loadMetadata.LoadProgresses.Any())
{
//Then the load is designed to run X days of source data at a time
//Load Progress
ILoadProgressSelectionStrategy whichLoadProgress = loadProgress != null ? (ILoadProgressSelectionStrategy) new SingleLoadProgressSelectionStrategy(loadProgress) : new AnyAvailableLoadProgressSelectionStrategy(loadMetadata);
var jobDateFactory = new JobDateGenerationStrategyFactory(whichLoadProgress);
dataLoadProcess = _options.Iterative
? (IDataLoadProcess) new IterativeScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration) :
new SingleJobScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration);
}
else
{
//OnDemand
dataLoadProcess = new DataLoadProcess(locator, loadMetadata, checkable, logManager, listener, execution, databaseConfiguration);
}
var exitCode = dataLoadProcess.Run(token);
//return 0 for success or load not required otherwise return the exit code (which will be non zero so error)
return(exitCode == ExitCodeType.Success || exitCode == ExitCodeType.OperationNotRequired? 0: (int)exitCode);
case CommandLineActivity.check:
checkable.Check(checkNotifier);
return(0);
default:
throw new ArgumentOutOfRangeException();
}
}
public ILoadCachePathResolver CreateResolver(ILoadProgress loadProgress)
{
throw new NotImplementedException();
}
public SingleLoadProgressSelectionStrategy(ILoadProgress loadProgress)
{
_loadProgress = loadProgress;
}
