public void RunTest()
{
using (var testFolder = new TestFolder())
{
var factory = new LoggerFactory();
var logger = factory.CreateLogger <RocksGraphProvider>();
var storeLogger = factory.CreateLogger <StoreProcessor>();
var provider = new RocksGraphProvider(logger, testFolder);
var storeProvider = new SetProvider(provider);
var storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeLogger);
var storeId = "test";
var points = new Dictionary <string, double>();
var pointCount = 0;
foreach (var pointId in _dataPoints)
{
pointCount++;
points.Add(pointId, pointCount + 0.234);
}
foreach (var id in _ids)
{
var json = JsonGenerator.GenerateTelemetry(id, points);
storeProcessor.PatchJson(storeId, json);
}
}
}
public void Setup()
{
_testFolder = new TestFolder();
while (_ids.Count < _maxIds)
{
_ids.Add(Guid.NewGuid().ToString());
}
while (_dataPoints.Count < _maxPoints)
{
_dataPoints.Add(Guid.NewGuid().ToString());
}
var factory = new LoggerFactory();
var logger = factory.CreateLogger <RocksGraphProvider>();
var storeLogger = factory.CreateLogger <StoreProcessor>();
var provider = new RocksGraphProvider(logger, _testFolder);
var storeProvider = new SetProvider(provider);
_storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeLogger);
}
public void Where <TResult>(string whereExpression, Expression <Func <TResult> > predicate)
{
_whereCause.Append($" ({whereExpression}) and");
var visitor = new SetProvider();
visitor.Visit(predicate);
//var dic = (IDictionary<string, object>)_dynamicModel;
var model = predicate.Compile().Invoke();
var types = model.GetType();
visitor.UpdatedFields.ForEach(t =>
{
var values = types.GetProperty(t.Parameter)?.GetValue(model);
_sqlPamater[t.Parameter] = values;
});
}
public IUpdateable <T> SetColumns <TResult>(Expression <Func <TResult> > predicate)
{
var setProvider = new SetProvider();
setProvider.Visit(predicate);
//var dic = (IDictionary<string, object>)_dynamicModel;
var model = predicate.Compile().Invoke();
var types = model.GetType();
setProvider.UpdatedFields.ForEach(t =>
{
var values = types.GetProperty(t.Parameter)?.GetValue(model);
setSql.Append($" {t.ColumnName} = {DbPrefix}{t.Parameter},");
SqlParameter[t.Parameter] = values;
});
return(this);
}
public static void RunTest(int appCount, int deviceCount, int devicePropertyCount, int sendCount, int senderThreadCount, bool tryOptimizeRocks)
{
Console.WriteLine("Creating Messages");
var apps = new List <string>(appCount);
var deviceIds = new List <string>(deviceCount);
var devicePropertyNames = new List <string>(devicePropertyCount);
var tasks = new List <Task>();
var sendQueue = new ConcurrentQueue <StoreEvent>();
while (apps.Count < appCount)
{
apps.Add(Guid.NewGuid().ToString());
}
while (deviceIds.Count < deviceCount)
{
deviceIds.Add(Guid.NewGuid().ToString());
}
while (devicePropertyNames.Count < devicePropertyCount)
{
devicePropertyNames.Add(Guid.NewGuid().ToString());
}
using (var testFolder = new TestFolder())
{
var factory = LoggerFactory.Create(builder => builder.AddConsole());
var logger = factory.CreateLogger <RocksGraphProvider>();
var storeLogger = factory.CreateLogger <StoreProcessor>();
var dbOptions = new DbOptions();
var provider = !tryOptimizeRocks ?
new RocksGraphProvider(logger, testFolder) :
new RocksGraphProvider(logger, testFolder, dbOptions.SetCreateIfMissing(true)
.SetAllowConcurrentMemtableWrite(true)
//.SetAllowMmapReads(true)
//.SetAllowMmapWrites(true)
//.SetUseFsync(0)
.IncreaseParallelism(Environment.ProcessorCount)
.SetMaxBackgroundCompactions(Environment.ProcessorCount)
.SetMaxBackgroundFlushes(Environment.ProcessorCount));
var storeProvider = new SetProvider(provider);
var storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeLogger);
var storeConfig = new StoreConfig();
var storeError = new StoreError();
var eventReceiver = new EventReceiver(storeProcessor, null, storeConfig, factory.CreateLogger <EventReceiver>());
var queueContainer = new QueueContainer(eventReceiver, factory.CreateLogger <QueueContainer>(), storeError, 1_000_000);
var eventSender = new EventSender(queueContainer, null, null, storeConfig);
for (var i = 0; i < sendCount; i++)
{
foreach (var id in deviceIds)
{
foreach (var app in apps)
{
var points = GetPropertyValues(devicePropertyNames, _random);
var e = new StoreEvent
{
Operation = EventType.PATCH_JSON,
Data = JsonGenerator.GenerateTelemetry(id, points),
PartitionId = id,
StoreId = app
};
sendQueue.Enqueue(e);
}
}
}
Console.WriteLine($"Starting send of {sendQueue.Count} messages");
var timer = Stopwatch.StartNew();
for (var i = 0; i < senderThreadCount; i++)
{
tasks.Add(Task.Run(() => RunSender(eventSender, sendQueue)));
}
Task.WhenAll(tasks).Wait();
Console.WriteLine($"Completed writing to queues in {timer.Elapsed}");
while (queueContainer.Count() > 0)
{
Thread.Sleep(1000);
}
Console.WriteLine($"Completed writing to storage in {timer.Elapsed}");
}
}
