public RecordMocks BuildRecordMocks(MonitorInfo monitorInfo)
{
var seeder = new Mock<IConfigSeed>();
seeder.Setup(x => x.Seed("Test", MonitorReductionType.DefaultAccumulate)).Returns(monitorInfo);
var settings = BuildSettings();
settings.SetupGet(x => x.ConfigSeed).Returns(seeder.Object);
var cache = new Mock<IDataCache>();
cache.SetupGet(x => x.Empty).Returns(new MonitorRecord<double>());
return new RecordMocks { Cache = cache, Settings = settings, Seeder = seeder };
}
public void ShouldUpdateExistingData()
{
var connection = new Mock<IDbConnection>();
var connectionInstance = connection.Object;
IEnumerable<MonitorRecord<double>> updateList = null;
var storeData = new List<MonitorRecord<double>>
{
new MonitorRecord<double>(new DateTime(2010, 6, 6, 10, 30, 0), 1),
new MonitorRecord<double>(new DateTime(2010, 6, 6, 10, 30, 1), 1)
};
var store = new Mock<IStorageCommands>();
store.Setup(x => x.SelectListForUpdateExisting("TestSecondlyData", new DateTime(2010, 6, 6, 10, 30, 0, 100), connectionInstance, null)).Returns(storeData).Verifiable();
store.Setup(x => x.Update("TestSecondlyData", It.IsAny<IEnumerable<MonitorRecord<double>>>(), connectionInstance, null)).Callback((string tableName, IEnumerable<MonitorRecord<double>> values, IDbConnection conn, IDbTransaction transaction) => { updateList = values; }).Verifiable();
var reduceLevel = new ReduceLevel { AggregationClass = new ReduceMethodAccumulate(), Resolution = 1000 };
var monitorInfo = new MonitorInfo { FirstReduceLevel = reduceLevel };
var dictionary = new ConcurrentDictionary<string, MonitorInfo>();
dictionary.TryAdd("Test", monitorInfo);
var cache = new Mock<IDataCache>();
cache.SetupGet(x => x.Empty).Returns(new MonitorRecord<double>()).Verifiable();
cache.SetupGet(x => x.MonitorInfo).Returns(dictionary).Verifiable();
var data = new Dictionary<long, MonitorRecord<double>>
{
{ 1000, new MonitorRecord<double>(new DateTime(2010, 6, 6, 10, 30, 0, 100), 3) },
{ 1001, new MonitorRecord<double>(new DateTime(2010, 6, 6, 10, 30, 0, 200), 5) },
{ 1002, new MonitorRecord<double>(new DateTime(2010, 6, 6, 10, 30, 0, 400), 1) },
{ 1003, new MonitorRecord<double>(new DateTime(2010, 6, 6, 10, 30, 1, 300), 2) }
};
var dataUpdate = new RecordFlushUpdate(cache.Object, store.Object);
var result = dataUpdate.UpdateExisting("Test", data, connectionInstance);
Assert.True(result);
//I'm really not sure about these assertions as given the test data the output isn't what I would expect
Assert.Equal(2, updateList.Count());
var first = updateList.First();
Assert.Equal(4, first.Value);
Assert.Equal(2, first.Number);
var last = updateList.Last();
Assert.Equal(3, last.Value);
Assert.Equal(2, last.Number);
cache.VerifyAll();
store.VerifyAll();
}
public void ShouldBeAbleToPullExistingInfoFromCache()
{
var monitorInfo = new MonitorInfo();
var monitorInfoDictionary = new ConcurrentDictionary<string, MonitorInfo>();
monitorInfoDictionary.TryAdd("Test", monitorInfo);
var cache = new Mock<IDataCache>();
cache.SetupGet(x => x.MonitorInfo).Returns(monitorInfoDictionary);
var settings = BuildSettings();
var seeder = new ConfigSeed(cache.Object, settings.Object);
var result = seeder.Seed("Test", MonitorReductionType.DefaultAccumulate);
Assert.NotNull(result);
Assert.Same(monitorInfo, result);
}
public void CalculateComparisons(string configName, MonitorInfo monitorInfo, ReduceLevel reduceLevel, IDbConnection conn)
{
try
{
if (monitorInfo.MonitorConfig.ComparisonCalculator != null)
{
var comparisons = new SortedDictionary<long, MonitorRecordComparison<double>>();
monitorInfo.MonitorConfig.ComparisonCalculator.CalculateExpectedValues(configName, reduceLevel, comparisons, conn);
var tableName = Support.MakeReducedName(configName, reduceLevel.Resolution);
_storageCommands.Insert(tableName, comparisons.Values, conn, null);
}
}
catch (Exception e)
{
_logger.Fatal("Exception swallowed: ", e);
if (_settings.Debug)
throw;
}
}
private IEnumerable<DeleteInfo> ProcessConfig(string configName, MonitorInfo monitorInfo, bool deleteReducedData, IDbConnection conn)
{
var deleteInfo = new List<DeleteInfo>();
ReduceLevel lastReduceLevel = null;
foreach (var targetReduceLevel in monitorInfo.MonitorConfig.ReduceLevels)
{
if (lastReduceLevel == null)
{
lastReduceLevel = targetReduceLevel;
continue;
}
//Get the most recent data point that's already reduced
var targetReducedTableName = Support.MakeReducedName(configName, targetReduceLevel.Resolution);
var targetReducedData = _storageCommands.RetrieveLastReducedData(targetReducedTableName, targetReduceLevel.Resolution, conn);
//Get the data to be reduced, starting from the last point that was already reduced
var toBeReducedTableName = Support.MakeReducedName(configName, lastReduceLevel.Resolution);
var toBeReducedData = _storageCommands.SelectListRequiringReduction(toBeReducedTableName, targetReducedData.Record != null, targetReducedData.Time, conn);
//Reduce the data down and aggregate it together
var aggregatedList = new Dictionary<DateTime, IList<MonitorRecord<double>>>();
var lastAggregatedTime = _reduceAggregater.Aggregate(targetReducedData.Time, targetReduceLevel, toBeReducedData, aggregatedList);
//Write it out the data
var reducedList = CalculateReductions(aggregatedList, targetReduceLevel);
if (reducedList.Count > 0)
{
var updated = _storageCommands.UpdateIfExists(targetReducedTableName, reducedList.First(), targetReducedData.Record != null, conn);
if (updated)
reducedList.RemoveAt(0);
_storageCommands.Flush(targetReducedTableName, reducedList, conn);
//Prepare to delete old data
if (deleteReducedData)
deleteInfo.Add(new DeleteInfo { ConfigName = configName, LastAggregatedTime = lastAggregatedTime, LastReduceLevel = lastReduceLevel });
}
lastReduceLevel = targetReduceLevel;
//Remove duplicate reduce data if any exists
_storageCommands.PergeDuplicateReducedData(targetReducedTableName, conn);
}
return deleteInfo;
}
private void OtherCalculations(string configName, MonitorInfo monitorInfo, IDbConnection conn)
{
//After we've calculated all the reductions, we can do calculations of other data
foreach (var targetReduceLevel in monitorInfo.MonitorConfig.ReduceLevels)
_recordCompare.CalculateComparisons(configName, monitorInfo, targetReduceLevel, conn);
}
/// <summary>
/// Seed the local data for a monitor configuration.
/// </summary>
/// <param name="configName"></param>
/// <param name="monitorReductionType"></param>
/// <returns></returns>
public MonitorInfo Seed(string configName, MonitorReductionType monitorReductionType)
{
configName.ThrowIfNull("configName");
monitorReductionType.ThrowIfNull("monitorReductionType");
MonitorInfo monitorInfo = null;
bool tablesCreated = false;
configName = Support.ValidateConfigName(configName);
// Look for static cached data/state (i.e. its already been seeded)
if (!_cache.MonitorInfo.TryGetValue(configName, out monitorInfo))
{
// Look for an existing monitor configuration
MonitorConfig monitorConfig;
if (!_cache.MonitorConfigs.TryGetValue(configName, out monitorConfig))
{
// Create new monitorconfig record (will get inserted into db during Flush())
var aggregationClassName = monitorReductionType == MonitorReductionType.DefaultAccumulate ? "ZocMonLib.ReduceMethods.ReduceMethodAccumulate" : "ZocMonLib.ReduceMethods.ReduceMethodAverage";
var aggregationClass = _settings.ReduceMethodProvider.Retrieve(aggregationClassName);
monitorConfig = new MonitorConfig
{
Name = configName,
MonitorReductionType = monitorReductionType,
ReduceLevels = new List<ReduceLevel>
{
new ReduceLevel
{
MonitorConfigName = configName,
Resolution = 60 * 1000,
HistoryLength = 7 * 24 * 60 * 60 * 1000,
AggregationClassName = aggregationClassName,
AggregationClass = aggregationClass
},
new ReduceLevel
{
MonitorConfigName = configName,
Resolution = 5 * 60 * 1000,
HistoryLength = 7 * 24 * 60 * 60 * 1000,
AggregationClassName = aggregationClassName,
AggregationClass = aggregationClass
},
new ReduceLevel
{
MonitorConfigName = configName,
Resolution = 60 * 60 * 1000,
HistoryLength = 7 * 24 * 60 * 60 * 1000,
AggregationClassName = aggregationClassName,
AggregationClass = aggregationClass
},
new ReduceLevel
{
MonitorConfigName = configName,
Resolution = 24 * 60 * 60 * 1000,
HistoryLength = 7 * 24 * 60 * 60 * 1000,
AggregationClassName = aggregationClassName,
AggregationClass = aggregationClass
}
}
};
}
else
{
// If this monitor already exists in monitorconfigs, we know the tables have already been created
tablesCreated = true;
}
// Don't validate custom configs (as they could be anything....)
if (!MonitorReductionType.Custom.Equals(monitorReductionType))
{
if (!monitorReductionType.Equals(monitorConfig.MonitorReductionType))
throw new DataException("Wrong reduction type for monitor \"" + configName + "\"");
}
// WTF why only the first
var reduceLevel = monitorConfig.ReduceLevels.First();
monitorInfo = new MonitorInfo
{
MonitorConfig = monitorConfig,
FirstReduceLevel = reduceLevel,
MonitorRecords = new List<MonitorRecord<double>>(),
TablesCreated = tablesCreated
};
_cache.MonitorInfo.AddOrUpdate(configName, monitorInfo, (key, oldValue) => monitorInfo);
}
return monitorInfo;
}
private static TestData PopulateTestData(bool isAll)
{
var testData = new TestData();
var reduceStatus = new Mock<IRecordReduceStatus>();
reduceStatus.Setup(x => x.IsReducing()).Returns(false).Verifiable();
reduceStatus.Setup(x => x.DoneReducing()).Verifiable();
var reductionLevel1 = new ReduceLevel { Resolution = 1000, AggregationClass = new ReduceMethodAccumulate() };
var reductionLevel2 = new ReduceLevel { Resolution = 60000, AggregationClass = new ReduceMethodAccumulate() };
var monitorConfigSingle = new MonitorConfig { Name = "Single", ReduceLevels = new List<ReduceLevel> { reductionLevel1, reductionLevel2 } };
var monitorInfoSingle = new MonitorInfo { MonitorConfig = monitorConfigSingle };
var monitorInfoDictionary = new ConcurrentDictionary<string, MonitorInfo>();
var monitorConfigDictionary = new ConcurrentDictionary<string, MonitorConfig>();
monitorConfigDictionary.TryAdd("Single", monitorConfigSingle);
var seeder = new Mock<IConfigSeed>();
seeder.Setup(x => x.Seed("Single", MonitorReductionType.Custom)).Returns(monitorInfoSingle).Verifiable();
var cache = new Mock<IDataCache>();
cache.SetupGet(x => x.MonitorInfo).Returns(monitorInfoDictionary).Verifiable();
if (isAll)
cache.SetupGet(x => x.MonitorConfigs).Returns(monitorConfigDictionary).Verifiable();
var connection = new Mock<IDbConnection>();
connection.Setup(x => x.Open()).Verifiable();
if (!isAll)
connection.Setup(x => x.Close()).Verifiable();
var connectionInstance = connection.Object;
var dbProviderFactory = new Mock<IStorageFactory>();
dbProviderFactory.Setup(x => x.CreateConnection()).Returns(connectionInstance).Verifiable();
var requiringReduction = new List<MonitorRecord<double>> { new MonitorRecord<double>(new DateTime(2012, 1, 15, 6, 30, 5), 1), new MonitorRecord<double>(new DateTime(2012, 1, 15, 6, 30, 10), 2), new MonitorRecord<double>(new DateTime(2012, 1, 15, 6, 30, 15), 4) };
var reducedRecord = new MonitorRecord<double>(new DateTime(2012, 1, 15, 6, 30, 30), 7);
var reduced = new StorageLastReduced { Record = reducedRecord, Time = new DateTime(2012, 1, 15, 6, 30, 0) };
var storage = new Mock<IStorageCommands>();
storage.Setup(x => x.RetrieveLastReducedData("SingleMinutelyData", 60000, connectionInstance)).Returns(reduced).Verifiable();
storage.Setup(x => x.SelectListRequiringReduction("SingleSecondlyData", true, new DateTime(2012, 1, 15, 6, 30, 0), connectionInstance)).Returns(requiringReduction).Verifiable();
storage.Setup(x => x.UpdateIfExists("SingleMinutelyData", It.IsAny<MonitorRecord<double>>(), true, connectionInstance)).Callback(() => testData.UpdateIfExistsCount++).Returns(true).Verifiable();
storage.Setup(x => x.Flush("SingleMinutelyData", It.IsAny<IEnumerable<MonitorRecord<double>>>(), connectionInstance)).Verifiable();
storage.Setup(x => x.ClearReducedData("Single", It.IsAny<DateTime>(), It.IsAny<ReduceLevel>(), connectionInstance)).Callback(() => testData.DeletedCount++).Verifiable();
var reduceAggregater = new Mock<IRecordReduceAggregate>();
reduceAggregater.Setup(x => x.Aggregate(new DateTime(2012, 1, 15, 6, 30, 0), reductionLevel2, requiringReduction, It.IsAny<IDictionary<DateTime, IList<MonitorRecord<double>>>>()))
.Callback(() => testData.AggregateCount++)
.Returns((DateTime lastReductionTime, ReduceLevel targetReduceLevel, IList<MonitorRecord<double>> sourceAggregationList, IDictionary<DateTime, IList<MonitorRecord<double>>> destinationAggregatedList) => { destinationAggregatedList.Add(new DateTime(2012, 1, 15, 6, 30, 0), requiringReduction); return new DateTime(2012, 1, 15, 6, 30, 0); });
var comparisonsData = new Mock<IRecordCompare>();
comparisonsData.Setup(x => x.CalculateComparisons("Single", monitorInfoSingle, It.IsAny<ReduceLevel>(), connectionInstance)).Callback(() => testData.CalculateCount++).Verifiable();
var settings = BuildSettings();
settings.Setup(x => x.ConfigSeed).Returns(seeder.Object);
testData.Settings = settings;
testData.Connection = connection;
testData.ReduceStatus = reduceStatus;
testData.ReduceAggregater = reduceAggregater;
testData.Cache = cache;
testData.Seeder = seeder;
testData.ComparisonsData = comparisonsData;
testData.Storage = storage;
testData.DbProviderFactory = dbProviderFactory;
return testData;
}
private MonitorInfo BuildDefaultMonitorInfo(long resolution)
{
var monitorRecord = new MonitorRecord<double>(new DateTime(2001, 06, 06, 7, 50, 15, 234), 5);
var reductionMethod = new Mock<IReduceMethod<double>>();
reductionMethod.Setup(x => x.IntervalAggregate(It.IsAny<DateTime>(), null, 1)).Returns(monitorRecord);
var monitorInfo = new MonitorInfo
{
MonitorRecords = new List<MonitorRecord<double>>(),
MonitorConfig = new MonitorConfig(),
FirstReduceLevel = new ReduceLevel
{
Resolution = resolution,
//AggregationClass = reductionMethod.Object
AggregationClass = new ReduceMethodAccumulate()
}
};
return monitorInfo;
}