private IDataFlowPipelineEngine CreateRetryCachingEngine(ICacheProgress cacheProgress)
{
var cachingPipelineEngineFactory = new CachingPipelineUseCase(cacheProgress, true, new FailedCacheFetchRequestProvider(cacheProgress));
var engine = cachingPipelineEngineFactory.GetEngine(_listener);
_engineMap.Add(engine, cacheProgress.LoadProgress);
return(engine);
}
private IDataFlowPipelineEngine CreateCachingEngine(ICacheProgress cacheProgress)
{
var cachingPipelineEngineFactory = new CachingPipelineUseCase(cacheProgress);
var engine = cachingPipelineEngineFactory.GetEngine(_listener);
_engineMap.Add(engine, cacheProgress.LoadProgress);
return(engine);
}
public Task Fetch(DateTime startDate, DateTime endDate, GracefulCancellationToken token, IDataLoadEventListener listener, bool ignorePermissionWindow = false)
{
var dateToRetrieve = new DateTime(startDate.Year, startDate.Month, startDate.Day);
var initialFetchRequest = new BackfillCacheFetchRequest(_catalogueRepository, dateToRetrieve)
{
CacheProgress = _cacheProgress,
ChunkPeriod = _cacheProgress.ChunkPeriod
};
var requestProvider = (startDate == endDate)
? (ICacheFetchRequestProvider) new SingleDayCacheFetchRequestProvider(initialFetchRequest)
: new MultiDayCacheFetchRequestProvider(initialFetchRequest, endDate);
var factory = new CachingPipelineUseCase(_cacheProgress, ignorePermissionWindow, requestProvider);
var engine = factory.GetEngine(listener);
return(Task.Factory.StartNew(() => engine.ExecutePipeline(token)));
}
private IDataFlowPipelineEngine CreateCachingEngine(ICacheProgress cacheProgress)
{
var cachingPipelineEngineFactory = new CachingPipelineUseCase(cacheProgress);
return(cachingPipelineEngineFactory.GetEngine(_listener));
}
public void Check(ICheckNotifier notifier)
{
try
{
if (_cacheProgress.Pipeline_ID == null)
{
throw new Exception("CacheProgress " + _cacheProgress.ID + " doesn't have a caching pipeline!");
}
IPipeline pipeline = null;
try
{
pipeline = _cacheProgress.Pipeline;
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Error when trying to load Pipeline ID = " + _cacheProgress.Pipeline_ID.Value, CheckResult.Fail, e));
}
if (pipeline == null)
{
notifier.OnCheckPerformed(new CheckEventArgs("Could not run Pipeline checks due to previous errors", CheckResult.Fail));
}
else
{
var checker = new PipelineChecker(pipeline);
checker.Check(notifier);
}
if (_cacheProgress.CacheFillProgress == null && _cacheProgress.LoadProgress.OriginDate == null)
{
//if we don't know what dates to request
notifier.OnCheckPerformed(
new CheckEventArgs(
"Both the CacheFillProgress and the LoadProgress.OriginDate are null, this means we don't know where the cache has filled up to and we don't know when the dataset is supposed to start. This means it is impossible to know what dates to fetch",
CheckResult.Fail));
}
if (_cacheProgress.PermissionWindow_ID != null && !_cacheProgress.PermissionWindow.WithinPermissionWindow(DateTime.UtcNow))
{
notifier.OnCheckPerformed(new CheckEventArgs(
"Current time is " + DateTime.UtcNow +
" which is not a permitted time according to the configured PermissionWindow " + _cacheProgress.PermissionWindow.Description +
" of the CacheProgress " + _cacheProgress,
CheckResult.Warning));
}
var shortfall = _cacheProgress.GetShortfall();
if (shortfall <= TimeSpan.Zero)
{
if (_cacheProgress.CacheLagPeriod == null)
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"CacheProgress reports that it has loaded up till " + _cacheProgress.CacheFillProgress +
" which is in the future. So we don't need to load this cache.", CheckResult.Warning));
}
else
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"CacheProgress reports that it has loaded up till " + _cacheProgress.CacheFillProgress +
" but there is a lag period of " + _cacheProgress.CacheLagPeriod +
" which means we are not due to load any cached data yet.", CheckResult.Warning));
}
}
var factory = new CachingPipelineUseCase(_cacheProgress);
IDataFlowPipelineEngine engine = null;
try
{
engine = factory.GetEngine(new FromCheckNotifierToDataLoadEventListener(notifier));
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Could not create IDataFlowPipelineEngine", CheckResult.Fail, e));
}
if (engine != null)
{
engine.Check(notifier);
}
}
catch (Exception e)
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Entire checking process for cache progress " + _cacheProgress +
" crashed, see Exception for details", CheckResult.Fail, e));
}
}