public void Query_Incoming_Level_Returns()
{
var(set, _, _) = StoreProcessor.GetGraphs("app2");
var query = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "contains", Value = "name"
}
},
PageSize = 10,
HasSubject = new LinkQuery[]
{
new LinkQuery
{
Path = "*",
Level = 3,
Target = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "root1"
}
}
}
}
}
};
var rsp = new ObjectQueryExecutor().Query(query, set);
var values = rsp.Values.Select(x => x.Subject).ToArray();
CollectionAssert.AreEqual(values, new string[] { "i00", "i01", "i02", "i03", "i04", "i05", "i06", "i07", "i08", "i09" });
Assert.AreEqual((Continuation) new Triple("i09", "name", TripleObject.FromData("name09")), rsp.Continuation);
}
public void Setup()
{
_testFolder = new TestFolder();
while (_ids.Count < _maxIds)
{
_ids.Add(Guid.NewGuid().ToString());
}
while (_dataPoints.Count < _maxPoints)
{
_dataPoints.Add(Guid.NewGuid().ToString());
}
var factory = new LoggerFactory();
var logger = factory.CreateLogger <RocksGraphProvider>();
var storeLogger = factory.CreateLogger <StoreProcessor>();
var provider = new RocksGraphProvider(logger, _testFolder);
var storeProvider = new SetProvider(provider);
var storeOperationFactory = new StoreOperationFactory();
_storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeOperationFactory, storeLogger);
}
public void Query_Value_Single_Returns()
{
var(set, _, _) = StoreProcessor.GetGraphs("app1");
var query = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "name500"
}
},
PageSize = 10
};
var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set);
var values = rsp.Values.ToArray();
var testTriple = new Triple("500", "name", (new JValue("name500"), -1));
CollectionAssert.Contains(values, testTriple);
Assert.AreEqual(1, rsp.Values.Count());
Assert.AreEqual(null, rsp.Continuation);
}
public void Query_Value_EQ_Multiple_Returns_Empty()
{
var(set, _, _) = StoreProcessor.GetGraphs("app1");
var query = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "name100"
},
["age"] = new QueryUnit {
Operator = "eq", Value = 12
}
},
PageSize = 10
};
var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set);
Assert.AreEqual(0, rsp.Values.Count());
Assert.AreEqual(rsp.Continuation, null);
}
public void Query_With_Incoming_Single_Level_Returns()
{
var(set, _, _) = StoreProcessor.GetGraphs("app1");
var query = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "name300"
}
},
PageSize = 10,
HasSubject = new LinkQuery[]
{
new LinkQuery
{
Path = "values",
Target = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "name200"
}
}
}
}
}
};
var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set);
var values = rsp.Values.Select(x => x.Subject).ToArray();
CollectionAssert.Contains(values, "300");
Assert.AreEqual(1, values.Length);
}
public void Query_With_Outgoing_EQ_Returns()
{
var(set, _, _) = StoreProcessor.GetGraphs("app1");
var query = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "name100"
}
},
PageSize = 10,
HasObject = new LinkQuery[]
{
new LinkQuery
{
Path = "contains/values",
Target = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "name300"
}
}
}
}
}
};
var rsp = new ObjectQueryExecutor().Query(query, set);
var values = rsp.Values.Select(x => x.Subject).ToArray();
CollectionAssert.Contains(values, "100");
Assert.AreEqual(1, rsp.Values.Count());
}
public void Query_Outgoing_Two_Level_Star_Returns()
{
var(set, _, _) = StoreProcessor.GetGraphs("app1");
var query = new ObjectQueryModel {
Filter = new Dictionary <string, QueryUnit>()
{
["type"] = new QueryUnit {
Operator = "eq", Value = "building"
}
},
PageSize = 10,
HasObject = new LinkQuery[]
{
new LinkQuery
{
Path = "*/*",
Target = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "Entity room 4"
}
}
}
}
}
};
var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set);
var values = rsp.Values.ToArray();
Assert.AreEqual(1, values.Count());
Assert.AreEqual(rsp.Continuation, null);
Assert.AreEqual("oBZ_JoNOBC", values.First().Subject);
}
public void Patch_Single_Value_Array_With_Triple_Returns()
{
var doc = new {
id = "100",
name = "Device 100",
otherName = "Other Name",
contains = new int[] { 20 }
};
var patch = new {
remove = new {
id = "100",
otherName = "Other Name",
contains = new int[] { 20 }
},
add = new {
id = "100",
contains = new int[] { 8 }
}
};
var expected = new {
id = "100",
name = "Device 100",
contains = new int[] { 8 }
};
StoreProcessor.Assert("app3", JObject.FromObject(doc), false);
StoreProcessor.PatchTriple("app3", JObject.FromObject(patch));
var rsp = StoreProcessor.GetSubject("app3", "100", null, 3);
var rspString = TripleConverter.FromJson(rsp).ToArray();
var expectedString = TripleConverter.FromJson(JObject.FromObject(expected)).ToArray();
CollectionAssert.AreEquivalent(rspString, expectedString);
}
public void Query_With_PageSize_Returns()
{
var(set, _, _) = StoreProcessor.GetGraphs("app1");
var query = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "contains", Value = "name"
}
},
PageSize = 10,
HasObject = new LinkQuery[]
{
new LinkQuery
{
Path = "contains/values",
Target = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "namev0"
}
}
}
}
}
};
var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set);
var values = rsp.Values.Select(x => x.Subject).ToArray();
CollectionAssert.AreEqual(values, new string[] { "i00", "i01", "i02", "i03", "i04", "i05", "i06", "i07", "i08", "i09" });
Assert.AreEqual(new Triple("i09", "name", TripleObject.FromData("name09")), rsp.Continuation);
var query2 = new ObjectQueryModel
{
Continuation = rsp.Continuation,
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "contains", Value = "name"
}
},
PageSize = 10,
HasObject = new LinkQuery[]
{
new LinkQuery
{
Path = "contains/values",
Target = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "namev0"
}
}
}
}
}
};
var rsp2 = new ObjectQueryExecutor().Query(query2, (RocksGraph)set);
var values2 = rsp2.Values.Select(x => x.Subject).ToArray();
CollectionAssert.AreEqual(values2, new string[] { "i10", "i11", "i12", "i13", "i14", "i15", "i16", "i17", "i18", "i19" });
Assert.AreEqual(new Triple("i19", "name", TripleObject.FromData("name19")), rsp2.Continuation);
var query3 = new ObjectQueryModel
{
Continuation = rsp2.Continuation,
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "contains", Value = "name"
}
},
PageSize = 1000,
HasObject = new LinkQuery[]
{
new LinkQuery
{
Path = "contains/values",
Target = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "eq", Value = "namev0"
}
}
}
}
}
};
var rsp3 = new ObjectQueryExecutor().Query(query3, (RocksGraph)set);
var values3 = rsp3.Values.Select(x => x.Subject).ToArray();
Assert.AreEqual(80, values3.Length);
Assert.AreEqual(null, rsp3.Continuation);
}
public void Query_With_Continuation_Returns()
{
var(set, _, _) = StoreProcessor.GetGraphs("app1");
var query = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "contains", Value = "name"
},
},
PageSize = 2
};
var rsp1 = new ObjectQueryExecutor().Query(query, (RocksGraph)set);
Assert.AreEqual(2, rsp1.Values.Count());
Assert.AreEqual(rsp1.Continuation, new Triple("200", "name", TripleObject.FromData("name200")));
var query2 = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "contains", Value = "name"
},
},
Continuation = rsp1.Continuation,
PageSize = 2
};
var rsp2 = new ObjectQueryExecutor().Query(query2, (RocksGraph)set);
Assert.AreEqual(2, rsp2.Values.Count());
Assert.AreEqual(rsp2.Continuation, new Triple("400", "name", TripleObject.FromData("name400")));
var query3 = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "contains", Value = "name"
},
},
Continuation = rsp2.Continuation,
PageSize = 2
};
var rsp3 = new ObjectQueryExecutor().Query(query3, (RocksGraph)set);
Assert.AreEqual(2, rsp3.Values.Count());
Assert.AreEqual(rsp3.Continuation, new Triple("600", "name", TripleObject.FromData("name600")));
var query4 = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "contains", Value = "name"
},
},
Continuation = rsp3.Continuation,
PageSize = 2
};
var rsp4 = new ObjectQueryExecutor().Query(query4, (RocksGraph)set);
Assert.AreEqual(2, rsp4.Values.Count());
Assert.AreEqual(rsp4.Continuation, new Triple("800", "name", TripleObject.FromData("name800")));
var query5 = new ObjectQueryModel
{
Filter = new Dictionary <string, QueryUnit>()
{
["name"] = new QueryUnit {
Operator = "contains", Value = "name"
},
},
Continuation = rsp4.Continuation,
PageSize = 2
};
var rsp5 = new ObjectQueryExecutor().Query(query5, (RocksGraph)set);
Assert.AreEqual(0, rsp5.Values.Count());
Assert.AreEqual(rsp5.Continuation, null);
}
public static void RunTest(int appCount, int deviceCount, int devicePropertyCount, int sendCount, int senderThreadCount, bool tryOptimizeRocks)
{
Console.WriteLine("Creating Messages");
var apps = new List <string>(appCount);
var deviceIds = new List <string>(deviceCount);
var devicePropertyNames = new List <string>(devicePropertyCount);
var tasks = new List <Task>();
var sendQueue = new ConcurrentQueue <StoreEvent>();
while (apps.Count < appCount)
{
apps.Add(Guid.NewGuid().ToString());
}
while (deviceIds.Count < deviceCount)
{
deviceIds.Add(Guid.NewGuid().ToString());
}
while (devicePropertyNames.Count < devicePropertyCount)
{
devicePropertyNames.Add(Guid.NewGuid().ToString());
}
using (var testFolder = new TestFolder())
{
var factory = LoggerFactory.Create(builder => builder.AddConsole());
var logger = factory.CreateLogger <RocksGraphProvider>();
var storeLogger = factory.CreateLogger <StoreProcessor>();
var dbOptions = new DbOptions();
var provider = !tryOptimizeRocks ?
new RocksGraphProvider(logger, testFolder) :
new RocksGraphProvider(logger, testFolder, dbOptions.SetCreateIfMissing(true)
.SetAllowConcurrentMemtableWrite(true)
//.SetAllowMmapReads(true)
//.SetAllowMmapWrites(true)
//.SetUseFsync(0)
.IncreaseParallelism(Environment.ProcessorCount)
.SetMaxBackgroundCompactions(Environment.ProcessorCount)
.SetMaxBackgroundFlushes(Environment.ProcessorCount));
var storeProvider = new SetProvider(provider);
var storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeLogger);
var storeConfig = new StoreConfig();
var storeError = new StoreError();
var eventReceiver = new EventReceiver(storeProcessor, null, storeConfig, factory.CreateLogger <EventReceiver>());
var queueContainer = new QueueContainer(eventReceiver, factory.CreateLogger <QueueContainer>(), storeError, 1_000_000);
var eventSender = new EventSender(queueContainer, null, null, storeConfig);
for (var i = 0; i < sendCount; i++)
{
foreach (var id in deviceIds)
{
foreach (var app in apps)
{
var points = GetPropertyValues(devicePropertyNames, _random);
var e = new StoreEvent
{
Operation = EventType.PATCH_JSON,
Data = JsonGenerator.GenerateTelemetry(id, points),
PartitionId = id,
StoreId = app
};
sendQueue.Enqueue(e);
}
}
}
Console.WriteLine($"Starting send of {sendQueue.Count} messages");
var timer = Stopwatch.StartNew();
for (var i = 0; i < senderThreadCount; i++)
{
tasks.Add(Task.Run(() => RunSender(eventSender, sendQueue)));
}
Task.WhenAll(tasks).Wait();
Console.WriteLine($"Completed writing to queues in {timer.Elapsed}");
while (queueContainer.Count() > 0)
{
Thread.Sleep(1000);
}
Console.WriteLine($"Completed writing to storage in {timer.Elapsed}");
}
}
public void Patch_Object_Array_With_Triple_Returns()
{
var doc = new {
id = "100",
name = "Device 100",
otherName = "Other Name",
contains = new object[] {
new
{
id = "nested0",
name = "Nested 0"
},
new
{
id = "nested1",
name = "Nested 1"
},
new
{
id = "nested2",
name = "Nested 2"
},
new
{
id = "nested3",
name = "Nested 3"
}
}
};
var patch = new {
remove = new {
id = "100",
otherName = "Other Name",
contains = new object[]
{
new
{
id = "nested1",
name = "Nested 1"
},
new
{
id = "nested2",
name = "Nested 2"
}
}
},
add = new {
id = "100",
contains = new object[]
{
new
{
id = "nested5",
name = "Nested 5"
},
new
{
id = "nested4",
name = "Nested 4"
}
}
}
};
var expected = new {
id = "100",
contains = new object[] {
new
{
id = "nested0",
name = "Nested 0"
},
new
{
id = "nested3",
name = "Nested 3"
},
new
{
id = "nested5",
name = "Nested 5"
},
new
{
id = "nested4",
name = "Nested 4"
},
},
name = "Device 100",
};
StoreProcessor.Assert("app3", JObject.FromObject(doc), false);
StoreProcessor.PatchTriple("app3", JObject.FromObject(patch));
var rsp = StoreProcessor.GetSubject("app3", "100", null, 3);
var rspString = TripleConverter.FromJson(rsp).ToArray();
var expectedString = TripleConverter.FromJson(JObject.FromObject(expected)).ToArray();
CollectionAssert.AreEqual(rspString, expectedString, new UnorderedTripleComparer());
}
public StoreProcessorTest()
{
_items = new object[10];
foreach (var i in Enumerable.Range(0, 10))
{
dynamic item = new {
id = i.ToString(),
name = $"name{i}",
age = i * 5
};
_items[i] = item;
}
StoreProcessor.Assert("app1", JToken.FromObject(_items), false);
dynamic item1 = new {
id = "100",
name = "name100",
age = 20,
contains = new {
id = "200",
name = "name200",
values = new dynamic[] {
new
{
id = "300",
name = "name300",
age = 30,
},
new
{
id = "400",
name = "name400"
}
}
}
};
dynamic item2 = new {
id = "500",
name = "name500",
age = 20,
contains = new {
id = "600",
name = "name600",
values = new dynamic[] {
new
{
id = "700",
name = "name700",
age = 30,
},
new
{
id = "800",
name = "name800",
contains = new
{
id = "600"
}
}
}
}
};
StoreProcessor.Assert("app2", JToken.FromObject(item1), false);
StoreProcessor.Assert("app2", JToken.FromObject(item2), false);
}
public TripleTest()
{
var(set, _, _) = StoreProcessor.GetGraphs(SetId);
_set = set;
}
public ObjectQueryIncomingTest()
{
dynamic item1 = new
{
id = "100",
name = "name100",
age = 20,
contains = new
{
id = "200",
name = "name200",
values = new dynamic[] {
new
{
id = "300",
name = "name300",
age = 30,
},
new
{
id = "400",
name = "name400"
}
}
}
};
dynamic item2 = new
{
id = "500",
name = "name500",
age = 20,
contains = new
{
id = "600",
name = "name600",
values = new dynamic[] {
new
{
id = "700",
name = "name700",
age = 30,
},
new
{
id = "800",
name = "name800",
contains = new
{
id = "600"
}
}
}
}
};
StoreProcessor.Assert("app1", JToken.FromObject(item1), false);
StoreProcessor.Assert("app1", JToken.FromObject(item2), false);
for (int i = 0; i < 100; i++)
{
var item = new
{
id = "root001",
name = "root1",
contains = new
{
id = "c1",
name = "namec1",
values = new
{
id = $"i{i.ToString("D2")}",
name = $"name{i.ToString("D2")}",
}
}
};
StoreProcessor.Assert("app2", JToken.FromObject(item), false);
}
for (int i = 0; i < 100; i++)
{
var item = new
{
id = "root002",
name = "root2",
contains = new
{
id = "c2",
name = "namec2",
values = new
{
id = $"j{i.ToString("D2")}",
name = $"name{i.ToString("D2")}",
}
}
};
StoreProcessor.Assert("app2", JToken.FromObject(item), false);
}
}
public ObjectQueryStarPathTest()
{
var text = File.ReadAllText(Path.Combine("TestInput", "region.json"));
StoreProcessor.Assert("app1", JToken.Parse(text), true);
}
public void Json_Read_Returns_Array()
{
var rsp = StoreProcessor.GetSubject("app1", "simplearray", null, 1);
Assert.IsTrue(JToken.DeepEquals(JToken.FromObject(_item1), rsp));
}
