private static CacheOptions GetCommand(ICacheProgress cp)
{
return(new CacheOptions()
{
CacheProgress = cp.ID
});
}
protected virtual ICacheLayout CreateCacheLayout(ICacheProgress cacheProgress, IDataLoadEventListener listener)
{
var pipelineFactory = new CachingPipelineUseCase(cacheProgress);
var destination = pipelineFactory.CreateDestinationOnly(listener);
return(destination.CreateCacheLayout());
}
public FailedCacheFetchRequestProvider(ICacheProgress cacheProgress, int batchSize = 50)
{
_cacheProgress = cacheProgress;
_batchSize = batchSize;
Current = null;
}
private IDataFlowPipelineEngine CreateRetryCachingEngine(ICacheProgress cacheProgress)
{
var cachingPipelineEngineFactory = new CachingPipelineUseCase(cacheProgress, true, new FailedCacheFetchRequestProvider(cacheProgress));
var engine = cachingPipelineEngineFactory.GetEngine(_listener);
_engineMap.Add(engine, cacheProgress.LoadProgress);
return(engine);
}
/// <summary>
/// Creates a CacheFetchRequest from a previous failure.
/// </summary>
/// <param name="cacheFetchFailure"></param>
/// <param name="cacheProgress"></param>
public CacheFetchRequest(ICacheFetchFailure cacheFetchFailure, ICacheProgress cacheProgress)
{
Start = cacheFetchFailure.FetchRequestStart;
CacheProgress = cacheProgress;
ChunkPeriod = cacheFetchFailure.FetchRequestEnd.Subtract(cacheFetchFailure.FetchRequestStart);
PermissionWindow = cacheProgress.PermissionWindow;
PreviousFailure = cacheFetchFailure;
}
private IDataFlowPipelineEngine CreateCachingEngine(ICacheProgress cacheProgress)
{
var cachingPipelineEngineFactory = new CachingPipelineUseCase(cacheProgress);
var engine = cachingPipelineEngineFactory.GetEngine(_listener);
_engineMap.Add(engine, cacheProgress.LoadProgress);
return(engine);
}
/// <summary>
/// Documents that a given cache fetch request was not succesfully executed e.g. the remote endpoint returned an error for that date range.
/// </summary>
/// <param name="repository"></param>
/// <param name="cacheProgress"></param>
/// <param name="start"></param>
/// <param name="end"></param>
/// <param name="e"></param>
public CacheFetchFailure(ICatalogueRepository repository, ICacheProgress cacheProgress, DateTime start, DateTime end, Exception e)
{
repository.InsertAndHydrate(this, new Dictionary <string, object>
{
{ "CacheProgress_ID", cacheProgress.ID },
{ "FetchRequestStart", start },
{ "FetchRequestEnd", end },
{ "ExceptionText", ExceptionHelper.ExceptionToListOfInnerMessages(e, true) },
{ "LastAttempt", DateTime.Now },
{ "ResolvedOn", DBNull.Value }
});
}
/// <summary>
/// Class for helping you to construct a caching pipeline engine instance with the correct context and initialization objects
/// </summary>
/// <param name="cacheProgress">The cache that will be run</param>
/// <param name="ignorePermissionWindow">Set to true to ignore the CacheProgress.PermissionWindow (if any)</param>
/// <param name="providerIfAny">The strategy for figuring out what dates to load the cache with e.g. failed cache fetches or new jobs from head of que?</param>
/// <param name="throwIfNoPipeline"></param>
public CachingPipelineUseCase(ICacheProgress cacheProgress, bool ignorePermissionWindow = false, ICacheFetchRequestProvider providerIfAny = null, bool throwIfNoPipeline = true)
{
_cacheProgress = cacheProgress;
_providerIfAny = providerIfAny;
//if there is no permission window or we are ignoring it
if (ignorePermissionWindow || cacheProgress.PermissionWindow_ID == null)
{
_permissionWindow = new SpontaneouslyInventedPermissionWindow(_cacheProgress);
}
else
{
_permissionWindow = cacheProgress.PermissionWindow;
}
if (_providerIfAny == null)
{
_providerIfAny = new CacheFetchRequestProvider(_cacheProgress)
{
PermissionWindow = _permissionWindow
};
}
_pipeline = _cacheProgress.Pipeline;
if (_pipeline == null && throwIfNoPipeline)
{
throw new Exception("CacheProgress " + _cacheProgress + " does not have a Pipeline configured on it");
}
AddInitializationObject(_cacheProgress.Repository);
// Get the LoadDirectory for the engine initialization
var lmd = _cacheProgress.LoadProgress.LoadMetadata;
if (string.IsNullOrWhiteSpace(lmd.LocationOfFlatFiles))
{
if (throwIfNoPipeline)
{
throw new Exception("LoadMetadata '" + lmd + "' does not have a Load Directory specified, cannot create ProcessingPipelineUseCase without one");
}
}
else
{
AddInitializationObject(new LoadDirectory(lmd.LocationOfFlatFiles));
}
AddInitializationObject(_providerIfAny);
AddInitializationObject(_permissionWindow);
GenerateContext();
}
public LoadProgressSummaryReport(LoadProgress loadProgress)
{
_loadProgress = loadProgress;
_loadMetadata = _loadProgress.LoadMetadata;
_cacheProgress = _loadProgress.CacheProgress;
try
{
dqeRepository = new DQERepository(loadProgress.CatalogueRepository);
}
catch (NotSupportedException)
{
dqeRepository = null;
}
}
public override void SetDatabaseObject(IActivateItems activator, CacheProgress databaseObject)
{
base.SetDatabaseObject(activator, databaseObject);
_cacheProgress = databaseObject;
CommonFunctionality.AddToMenu(new ExecuteCommandEditCacheProgress(activator, databaseObject), "Edit");
CommonFunctionality.AddToMenu(new ExecuteCommandShowCacheFetchFailures(activator, databaseObject), "View Cache Failures");
bool failures = _cacheProgress.CacheFetchFailures.Any(f => f.ResolvedOn == null);
cbFailures.Enabled = failures;
checkAndExecuteUI1.SetItemActivator(activator);
}
private void PopulateCacheProgressPanel(ICacheProgress cacheProgress)
{
_bLoading = true;
SetCacheProgressTextBox();
var cacheLagPeriod = cacheProgress.GetCacheLagPeriod();
if (cacheLagPeriod != null)
{
udCacheLagDuration.Value = cacheLagPeriod.Duration;
ddCacheLagDurationType.SelectedItem = cacheLagPeriod.Type;
}
tbChunkPeriod.Text = cacheProgress.ChunkPeriod.ToString();
UpdateCacheLagPeriodControl();
SetupPipelineUI();
_bLoading = false;
}
public void FireItUpManually()
{
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataWriter));
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataInventor));
var cachingHost = new CachingHost(CatalogueRepository);
var cpAsList = new ICacheProgress[] { _cp }.ToList();
cachingHost.CacheProgressList = cpAsList;
cachingHost.Start(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
// should be numDaysToCache days in cache
Assert.AreEqual(NumDaysToCache, _LoadDirectory.Cache.GetFiles("*.csv").Count());
// make sure each file is named as expected
var cacheFiles = _LoadDirectory.Cache.GetFiles().Select(fi => fi.Name).ToArray();
for (var i = -NumDaysToCache; i < 0; i++)
{
var filename = DateTime.Now.AddDays(i).ToString("yyyyMMdd") + ".csv";
Assert.IsTrue(cacheFiles.Contains(filename), filename + " not found");
}
}
/// <summary>
/// Retrieves the destination component from the caching pipeline associated with the ICacheProgress object. The destination component is required to be an ICacheFileSystemDestination.
/// </summary>
/// <param name="cacheProgress"></param>
/// <returns></returns>
/// <exception cref="InvalidOperationException">Caching pipeline is not configured properly/doesn't exist</exception>
private static ICacheFileSystemDestination GetCacheDestinationPipelineComponent(ICacheProgress cacheProgress)
{
if (cacheProgress.Pipeline_ID == null)
{
throw new InvalidOperationException("This CacheProgress does not have a caching pipeline, please configure one.");
}
var factory = new CachingPipelineUseCase(cacheProgress);
ICacheFileSystemDestination destination;
try
{
destination = factory.CreateDestinationOnly(new ThrowImmediatelyDataLoadEventListener());
}
catch (Exception e)
{
throw new Exception("We identified that your cache uses pipeline " + cacheProgress.Pipeline + " but we could not instantiate the Pipeline's Destination instance, make sure the pipeline is intact in PipelineDiagramUI. See inner exception for details", e);
}
return(destination);
}
public CachedFileRetrieverTests()
{
_cpMock = Mock.Of <ICacheProgress>();
_lpMock = Mock.Of <ILoadProgress>(l => l.CacheProgress == _cpMock);
}
protected override ICacheLayout CreateCacheLayout(ICacheProgress cacheProgress, IDataLoadEventListener listener)
{
return(Layout);
}
public SpontaneouslyInventedPermissionWindow(ICacheProgress cp, List <PermissionWindowPeriod> windows) : this()
{
_cp = cp;
PermissionWindowPeriods = windows;
}
public SpontaneouslyInventedPermissionWindow(ICacheProgress cp) : this()
{
_cp = cp;
PermissionWindowPeriods = new List <PermissionWindowPeriod>();
}
public CachingPreExecutionChecker(ICacheProgress cacheProgress)
{
_cacheProgress = cacheProgress;
}
public RunCacheWindow(IBasicActivateItems activator, ICacheProgress cp)
: base(activator, () => GetCommand(cp))
{
}
public CustomDateCaching(ICacheProgress cacheProgress, ICatalogueRepository catalogueRepository)
{
_cacheProgress = cacheProgress;
_catalogueRepository = catalogueRepository;
}
/// <summary>
/// Sets up the class to generate <see cref="ICacheFetchRequest"/> for the given <see cref="ICacheProgress"/>
/// </summary>
/// <param name="cacheProgress">The cache which the request is for, this must have either an <see cref="ICacheProgress.CacheFillProgress"/> or it's
/// <see cref="ILoadProgress"/> parent must have a populated <see cref="ILoadProgress.OriginDate"/></param>
public CacheFetchRequestProvider(ICacheProgress cacheProgress)
{
CacheProgress = cacheProgress;
PermissionWindow = CacheProgress.PermissionWindow;
}
private IDataFlowPipelineEngine CreateCachingEngine(ICacheProgress cacheProgress)
{
var cachingPipelineEngineFactory = new CachingPipelineUseCase(cacheProgress);
return(cachingPipelineEngineFactory.GetEngine(_listener));
}
