public void PointAtEndTest2()
{
// nothing interesting happens here
var inputList = new[] {
StreamEvent.CreateInterval(1, 5, "A"),
StreamEvent.CreateInterval(2, 10, "A"),
StreamEvent.CreateInterval(3, 8, "A"),
StreamEvent.CreateInterval(4, 6, "A"),
StreamEvent.CreateInterval(8, 9, "A"),
};
var compareTo = new[] {
StreamEvent.CreatePoint(5, "A"),
StreamEvent.CreatePoint(6, "A"),
StreamEvent.CreatePoint(8, "A"),
StreamEvent.CreatePoint(9, "A"),
StreamEvent.CreatePoint(10, "A"),
END
};
var inputObservable = inputList.ToList().ToObservable();
var container = new QueryContainer();
var input = container.RegisterInput(inputObservable);
var outputStream = input.PointAtEnd();
var output = container.RegisterOutput(outputStream);
var result = new List <StreamEvent <string> >();
output.Subscribe(t => result.Add(t));
container.Restore(null);
Assert.IsTrue(result.SequenceEqual(compareTo));
}
public void PointAtEndTest4()
{
var payload = StructTuple.Create("A", 3);
// nothing interesting happens here
var inputList = new[] {
StreamEvent.CreateStart(1, payload),
StreamEvent.CreateEnd(2, 1, payload),
StreamEvent.CreateStart(2, payload),
StreamEvent.CreateEnd(3, 2, payload),
StreamEvent.CreateStart(3, payload),
StreamEvent.CreateEnd(4, 3, payload),
StreamEvent.CreateStart(4, payload),
StreamEvent.CreateEnd(5, 4, payload)
};
var compareTo = new StreamEvent <StructTuple <string, int> >[] {
StreamEvent.CreatePoint(2, payload),
StreamEvent.CreatePoint(3, payload),
StreamEvent.CreatePoint(4, payload),
StreamEvent.CreatePoint(5, payload),
END2
};
var input = inputList.ToList().ToObservable().ToStreamable().SetProperty().IsIntervalFree(true);
var outputStream = input.PointAtEnd();
Assert.IsTrue(outputStream.Select(r => r.Item1).IsEquivalentTo(compareTo.Select(r => r.IsData ? StreamEvent.CreatePoint(r.SyncTime, r.Payload.Item1) : END).ToArray()));
}
public void PointAtEndTest5()
{
var payload = StructTuple.Create("A", 3);
var inputList = new[]
{
StreamEvent.CreateInterval(1, 5, payload),
StreamEvent.CreateInterval(2, 10, payload),
StreamEvent.CreateInterval(3, 8, payload),
StreamEvent.CreateInterval(4, 6, payload),
StreamEvent.CreateInterval(8, 9, payload),
};
var compareTo = new StreamEvent <StructTuple <string, int> >[]
{
StreamEvent.CreatePoint(5, payload),
StreamEvent.CreatePoint(6, payload),
StreamEvent.CreatePoint(8, payload),
StreamEvent.CreatePoint(9, payload),
StreamEvent.CreatePoint(10, payload),
END2
};
var input = inputList.ToList().ToObservable().ToStreamable();
var outputStream = input.PointAtEnd();
Assert.IsTrue(outputStream.Select(r => r.Item1).IsEquivalentTo(compareTo.Select(r => r.IsData ? StreamEvent.CreatePoint(r.SyncTime, r.Payload.Item1) : END).ToArray()));
}
public void SurrogateTest()
{
var qc = new QueryContainer(new MySurrogate());
var output1 = new List <StreamEvent <IMyInterface> >();
var input = new Subject <StreamEvent <IMyInterface> >();
var ingress = qc.RegisterInput(input);
var egress = qc.RegisterOutput(ingress).ForEachAsync(o => output1.Add(o));
var process = qc.Restore();
input.OnNext(StreamEvent.CreatePoint(1, (IMyInterface) new MyType(1)));
var stream = new MemoryStream();
process.Checkpoint(stream);
stream.Position = 0;
input.OnCompleted();
var input2 = new Subject <StreamEvent <IMyInterface> >();
var output2 = new List <StreamEvent <IMyInterface> >();
var qc2 = new QueryContainer(new MySurrogate());
var ingress2 = qc2.RegisterInput(input2);
var egress2 = qc2.RegisterOutput(ingress2).ForEachAsync(o => output2.Add(o));
var process2 = qc2.Restore(stream);
input2.OnCompleted();
Assert.AreEqual(2, output2.Count);
Assert.AreEqual(1, output2[0].Payload.GetValue());
Assert.AreEqual(StreamEvent.InfinitySyncTime, output2[1].SyncTime);
}
private static void CreateSourceData()
{
source = new StreamEvent <Payload>[] {
StreamEvent.CreatePoint(100, new Payload {
Field1 = "A", Field2 = 4
}),
StreamEvent.CreatePoint(110, new Payload {
Field1 = "C", Field2 = 3
}),
StreamEvent.CreatePoint(120, new Payload {
Field1 = "A", Field2 = 1
}),
StreamEvent.CreatePoint(130, new Payload {
Field1 = "B", Field2 = 6
}),
StreamEvent.CreatePoint(140, new Payload {
Field1 = "B", Field2 = 8
}),
StreamEvent.CreatePoint(150, new Payload {
Field1 = "C", Field2 = 7
}),
StreamEvent.CreatePoint(160, new Payload {
Field1 = "B", Field2 = 9
}),
}.ToObservable().ToStreamable().AlterEventDuration(1000).Cache();
}
public void PointAtEndTest6()
{
var payload = StructTuple.Create("A", 3);
var inputList = new[]
{
StreamEvent.CreateStart(1, payload),
StreamEvent.CreateEnd(2, 1, payload),
StreamEvent.CreateStart(2, payload),
StreamEvent.CreateEnd(3, 2, payload),
StreamEvent.CreateStart(3, payload),
StreamEvent.CreateEnd(4, 3, payload),
StreamEvent.CreateStart(4, payload),
StreamEvent.CreateEnd(5, 4, payload)
};
var compareTo = new StreamEvent <StructTuple <string, int> >[]
{
StreamEvent.CreatePoint(101, payload),
StreamEvent.CreatePoint(102, payload),
StreamEvent.CreatePoint(103, payload),
StreamEvent.CreatePoint(104, payload),
END2
};
var input = inputList.ToList().ToObservable().ToStreamable()
.AlterEventDuration(100);
var outputStream = input.PointAtEnd();
Assert.IsTrue(outputStream.Select(r => r.Item1).IsEquivalentTo(compareTo.Select(r => r.IsData ? StreamEvent.CreatePoint(r.SyncTime, r.Payload.Item1) : END).ToArray()));
}
public Task StartObserving()
{
return(Task.Run(async() =>
{
while (1 == 1)
{
var result = await consumer.ReadAsync();
if (result.EndOfStream)
{
foreach (var observer in observers)
{
observer.OnCompleted();
}
return;
}
else
{
var streamEvent = StreamEvent.CreatePoint <T>(
startTime: result.MetaData.Timestamp.Ticks,
result.MessageData);
foreach (var observer in observers)
{
observer.OnNext(streamEvent);
}
}
}
}));
}
public void PointAtEndTest3()
{
// nothing interesting happens here
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(2, 1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateEnd(3, 2, "A"),
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateEnd(4, 3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(5, 4, "A")
};
var compareTo = new[]
{
StreamEvent.CreatePoint(101, "A"),
StreamEvent.CreatePoint(102, "A"),
StreamEvent.CreatePoint(103, "A"),
StreamEvent.CreatePoint(104, "A"),
END
};
var input = inputList.ToList().ToObservable().ToStreamable()
.AlterEventDuration(100);
var outputStream = input.PointAtEnd();
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
public void PointAtEndTest1()
{
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(2, 1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateEnd(3, 2, "A"),
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateEnd(4, 3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(5, 4, "A")
};
var compareTo = new[]
{
StreamEvent.CreatePoint(2, "A"),
StreamEvent.CreatePoint(3, "A"),
StreamEvent.CreatePoint(4, "A"),
StreamEvent.CreatePoint(5, "A"),
END
};
var input = inputList.ToList().ToObservable().ToStreamable().SetProperty().IsIntervalFree(true);
var outputStream = input.PointAtEnd();
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
private static void PopulateArray(StreamEvent <Event>[] array)
{
for (int i = 0; i < ArraySize - 1; i++)
{
var ev = Event.CreateEvent(DataGenerator.Ads);
array[i] = StreamEvent.CreatePoint(ev.event_time.Ticks, ev);
}
}
static void CreateData()
{
source1 = new StreamEvent <Payload>[] {
StreamEvent.CreatePoint(100, new Payload {
Field1 = "A", Field2 = 4
}),
StreamEvent.CreatePoint(110, new Payload {
Field1 = "C", Field2 = 3
}),
StreamEvent.CreatePoint(120, new Payload {
Field1 = "A", Field2 = 1
}),
StreamEvent.CreatePoint(130, new Payload {
Field1 = "B", Field2 = 6
}),
StreamEvent.CreatePoint(140, new Payload {
Field1 = "B", Field2 = 8
}),
StreamEvent.CreatePoint(150, new Payload {
Field1 = "C", Field2 = 7
}),
StreamEvent.CreatePoint(160, new Payload {
Field1 = "B", Field2 = 9
}),
}.ToObservable().ToStreamable(OnCompletedPolicy.EndOfStream()).AlterEventDuration(1000).Cache();
source2 = new StreamEvent <Payload>[] {
StreamEvent.CreatePoint(100, new Payload {
Field1 = "A", Field2 = 1
}),
StreamEvent.CreatePoint(110, new Payload {
Field1 = "C", Field2 = 2
}),
StreamEvent.CreatePoint(120, new Payload {
Field1 = "A", Field2 = 2
}),
StreamEvent.CreatePoint(130, new Payload {
Field1 = "B", Field2 = 2
}),
StreamEvent.CreatePoint(140, new Payload {
Field1 = "B", Field2 = 2
}),
StreamEvent.CreatePoint(150, new Payload {
Field1 = "C", Field2 = 1
}),
StreamEvent.CreatePoint(160, new Payload {
Field1 = "B", Field2 = 1
}),
}.ToObservable().ToStreamable(OnCompletedPolicy.EndOfStream()).AlterEventDuration(1000).Cache();
}
/*
* Mostly a normal blocking socket listener
* calls observer.onNext()
*/
public void StartListener()
{
byte[] bytes;
IPAddress ipAddress = IPAddress.Parse(ip);
IPEndPoint localEndpoint = new IPEndPoint(ipAddress, port);
observers = new List <IObserver <StreamEvent <Payload> > >();
// make blocking socket server
Socket server = new Socket(ipAddress.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
Console.WriteLine(localEndpoint.ToString());
try {
server.Bind(localEndpoint);
server.Listen(10);
while (true)
{
Console.WriteLine("Waiting for a connection...");
Socket handler = server.Accept();
string bytesString = null;
bytes = new byte[1024];
while (true)
{
int bytesRec = handler.Receive(bytes);
if (bytesRec > 0)
{
bytesString += Encoding.ASCII.GetString(bytes, 0, bytesRec);
}
else
{
break;
}
}
// Show the data on the console.
foreach (IObserver <StreamEvent <Payload> > observer in observers)
{
Payload payload = ParseInput(bytesString);
observer.OnNext(StreamEvent.CreatePoint(payload.Timestamp.Ticks, payload));
observer.OnNext(StreamEvent.CreatePunctuation <Payload>(payload.Timestamp.Ticks + 2));
}
}
} catch (Exception e) {
Console.WriteLine(e.ToString());
}
}
public void DisjointUnionPunctuations()
{
var left = new Subject <StreamEvent <int> >();
var right = new Subject <StreamEvent <int> >();
var qc = new QueryContainer();
var leftInput = qc.RegisterInput(left);
var rightInput = qc.RegisterInput(right);
var actualOutput = new List <StreamEvent <int> >();
var union = new MultiUnionStreamable <Empty, int>(new IStreamable <Empty, int>[] { leftInput, rightInput }, guaranteedDisjoint: true);
var egress = qc.RegisterOutput(union).ForEachAsync(o => actualOutput.Add(o));
var process = qc.Restore();
left.OnNext(StreamEvent.CreatePoint(100, 1));
left.OnNext(StreamEvent.CreatePunctuation <int>(101));
right.OnNext(StreamEvent.CreatePoint(100, 1));
right.OnNext(StreamEvent.CreatePunctuation <int>(110));
process.Flush();
left.OnNext(StreamEvent.CreatePoint(101, 1));
right.OnNext(StreamEvent.CreatePoint(110, 1));
process.Flush();
left.OnCompleted();
right.OnCompleted();
var expected = new StreamEvent <int>[]
{
StreamEvent.CreatePoint(100, 1),
StreamEvent.CreatePoint(100, 1),
StreamEvent.CreatePunctuation <int>(101),
StreamEvent.CreatePoint(101, 1),
StreamEvent.CreatePoint(110, 1),
StreamEvent.CreatePunctuation <int>(110),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime),
};
Assert.IsTrue(expected.SequenceEqual(actualOutput));
}
private static void JoinPointsTest(bool fixedInterval = false)
{
var left = new Subject <StreamEvent <string> >();
var right = new Subject <StreamEvent <string> >();
var qc = new QueryContainer();
IStreamable <Empty, string> leftInput = qc.RegisterInput(left);
IStreamable <Empty, string> rightInput = qc.RegisterInput(right);
if (fixedInterval)
{
leftInput = leftInput.AlterEventDuration(1);
rightInput = rightInput.AlterEventDuration(1);
}
var query = leftInput.Join(
rightInput,
l => (l != null ? l[0].ToString() : null),
r => (r != null ? r[0].ToString() : null),
(l, r) => $"{l},{r}");
var output = new List <StreamEvent <string> >();
qc.RegisterOutput(query).ForEachAsync(o => output.Add(o));
var process = qc.Restore();
// Should match and egress immediately
left.OnNext(StreamEvent.CreatePoint(100, "A1"));
right.OnNext(StreamEvent.CreatePoint(100, "A2"));
process.Flush();
var expected = new StreamEvent <string>[]
{
StreamEvent.CreatePoint(100, "A1,A2"),
};
Assert.IsTrue(expected.SequenceEqual(output));
output.Clear();
left.OnCompleted();
right.OnCompleted();
}
public void Stitch_PointsCodegen()
{
StreamEvent <int>[] input = new[]
{
StreamEvent.CreatePoint(100, 1),
StreamEvent.CreatePoint(102, 1),
};
IList <StreamEvent <int> > result = Execute(input, x => x.Stitch());
StreamEvent <int>[] expected = new[]
{
StreamEvent.CreateStart(100, 1),
StreamEvent.CreateEnd(101, 100, 1),
StreamEvent.CreateStart(102, 1),
StreamEvent.CreateEnd(103, 102, 1),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime),
};
Assert.IsTrue(expected.SequenceEqual(result));
}
/*
* Mostly a normal blocking socket listener
* calls observer.onNext()
*/
public void StartListener()
{
string[] topics =
{
"all-event-data-test"
};
var consumerConfig = new ConsumerConfig {
BootstrapServers = ip + ":" + port.ToString(),
EnableAutoCommit = false,
GroupId = "1"
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = "localhost:8081"
};
const int commitPeriod = 5;
observers = new List <IObserver <StreamEvent <Payload> > >();
CancellationTokenSource cts = new CancellationTokenSource();
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer = new ConsumerBuilder <Ignore, EventRecord>(consumerConfig)
.SetValueDeserializer(new AvroDeserializer <EventRecord>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build()) {
consumer.Subscribe(topics);
try {
while (true)
{
try {
var consumeResult = consumer.Consume(cts.Token);
var message = consumeResult.Message.Value;
foreach (IObserver <StreamEvent <Payload> > observer in observers)
{
Payload payload = new Payload(message.DeviceID, message.Timestamp, (short)message.EventID, new Location(message.EventData.Latitude, message.EventData.Longitude, message.EventData.Accuracy));
observer.OnNext(StreamEvent.CreatePoint(payload.Timestamp.Ticks, payload));
observer.OnNext(StreamEvent.CreatePunctuation <Payload>(payload.Timestamp.Ticks + 2));
}
if (consumeResult.Offset % commitPeriod == 0)
{
try {
consumer.Commit(consumeResult);
} catch (KafkaException e) {
Console.WriteLine($"Commit error: {e.Error.Reason}");
}
}
} catch (ConsumeException e) {
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
} catch (OperationCanceledException e) {
consumer.Close();
}
}
cts.Cancel();
}
public void LOJ1Row()
{
var container = new QueryContainer(null);
var left = new StreamEvent <MyData>[]
{
StreamEvent.CreatePoint(10, new MyData {
field1 = 1, field2 = "A"
}),
StreamEvent.CreatePoint(10, new MyData {
field1 = 1, field2 = "B"
}),
StreamEvent.CreatePoint(10, new MyData {
field1 = 2, field2 = "D"
}),
StreamEvent.CreatePoint(10, new MyData {
field1 = 2, field2 = "E"
})
};
var right = new StreamEvent <MyData2>[]
{
StreamEvent.CreatePoint(10, new MyData2 {
field3 = 1, field4 = "W"
}),
StreamEvent.CreatePoint(10, new MyData2 {
field3 = 1, field4 = "X"
}),
StreamEvent.CreatePoint(10, new MyData2 {
field3 = 2, field4 = "Y"
})
};
var output = new List <StreamEvent <MyData3> >();
var expected = new StreamEvent <MyData3>[]
{
StreamEvent.CreatePoint(10, new MyData3 {
field1 = 2, field2 = "E", field3 = -1, field4 = "null"
}),
StreamEvent.CreatePoint(10, new MyData3 {
field1 = 1, field2 = "B", field3 = 1, field4 = "W"
}),
StreamEvent.CreatePoint(10, new MyData3 {
field1 = 1, field2 = "A", field3 = 1, field4 = "W"
}),
StreamEvent.CreatePoint(10, new MyData3 {
field1 = 1, field2 = "B", field3 = 1, field4 = "X"
}),
StreamEvent.CreatePoint(10, new MyData3 {
field1 = 1, field2 = "A", field3 = 1, field4 = "X"
}),
StreamEvent.CreatePoint(10, new MyData3 {
field1 = 2, field2 = "D", field3 = 2, field4 = "Y"
}),
};
var leftStream = container.RegisterInput(left.ToObservable());
var rightStream = container.RegisterInput(right.ToObservable());
int tmp1 = -1;
string tmp2 = "null";
var query =
leftStream.LeftOuterJoin(rightStream, e => e.field1, e => e.field3,
(l, r) => l.field2 != "E",
(l) => new MyData3 {
field1 = l.field1, field2 = l.field2, field3 = tmp1, field4 = tmp2
},
(l, r) => new MyData3 {
field1 = l.field1, field2 = l.field2, field3 = r.field3, field4 = r.field4
});
var result = container.RegisterOutput(query, ReshapingPolicy.CoalesceEndEdges).Where(e => e.IsData);
var resultAsync = result.ForEachAsync(o => output.Add(o));
container.Restore(null); // start the query
Assert.IsTrue(output.ToArray().SequenceEqual(expected));
}
public static StreamEvent <T> ToStreamEvent <T>(this EffluxMessage message)
{
return(StreamEvent.CreatePoint <T>(
startTime: message.MetaData.Timestamp.Ticks,
message.PayloadAs <T>()));
}
private void ProduceEvents()
{
try
{
using (var streamReader = this._twitterStreaming.Read())
{
// Loop until stop signal
while (!_disposed)
{
try
{
// Read from source
var line = streamReader.ReadLine();
if (string.IsNullOrEmpty(line))
{
continue;
}
// Parse
var jObject = JObject.Parse(line);
string text = Unquote(jObject.SelectToken("text").Value <string>());
string topic = TextAnalysis.DetermineTopc(text, _config.twitter_keywords);
string language = (null == jObject.SelectToken("user.lang")) ? string.Empty : jObject.SelectToken("user.lang").Value <string>();
// filter out tweets we don't care about: non-english tweets or deleted tweets
if ((jObject["delete"] != null) ||
(language != "en") ||
(topic == "Unknown"))
{
continue;
}
var tweet = new Tweet();
tweet.ID = jObject.SelectToken("id_str").Value <Int64>();
tweet.CreatedAt = ParseTwitterDateTime(jObject.SelectToken("created_at").Value <string>());
tweet.UserName = Unquote(jObject.SelectToken("user.screen_name").Value <string>());
tweet.ProfileImageUrl = jObject.SelectToken("user.profile_image_url").Value <string>();
tweet.Text = text;
tweet.Topic = topic;
tweet.RawJson = line;
tweet.SentimentScore = this.Sentiment(text);
// Produce INSERT event
Observer.OnNext(StreamEvent.CreatePoint(tweet.CreatedAt.Ticks, tweet));
}
catch (Exception ex)
{
// Error handling should go here
Console.WriteLine("EXCEPTION RAISED in TwitterInputAdapter: " + ex.Message);
}
}
}
}
catch (Exception e)
{
Observer.OnError(e);
TwitterStream.Dispose(_config);
}
}
