public void OnStreamInstanceEvent(SyncedData <StreamInstance> streamInstance, StreamEvent streamEvent)
{
if (streamEvent == StreamEvent.Added)
{
OnStreamAdded(streamInstance);
}
}
public static EventItem FromStreamEvent(ISubscription sub, StreamEvent e)
{
return(new StreamEventItem(sub, e));
}
private static void LeftAntiSemiJoinEdgesTest()
{
var input1 = new[]
{
StreamEvent.CreateStart(100, "A1"),
StreamEvent.CreateEnd(110, 100, "A1"),
StreamEvent.CreateStart(101, "A2"),
StreamEvent.CreateEnd(105, 101, "A2"),
StreamEvent.CreateStart(102, "B1"),
StreamEvent.CreateEnd(112, 102, "B1"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
var input2 = new[]
{
StreamEvent.CreateStart(101, "A"),
StreamEvent.CreateEnd(103, 101, "A"),
StreamEvent.CreateStart(106, "A"),
StreamEvent.CreateEnd(108, 106, "A"),
StreamEvent.CreateStart(106, "A"),
StreamEvent.CreateEnd(109, 106, "A"),
StreamEvent.CreateStart(100, "C"),
StreamEvent.CreateEnd(110, 100, "C"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
var inputStream1 = input1.ToCleanStreamable();
var inputStream2 = input2.ToCleanStreamable();
var outputStream = inputStream1.WhereNotExists(
inputStream2,
l => (l != null ? l[0].ToString() : null),
r => (r != null ? r[0].ToString() : null));
var correct = new[]
{
StreamEvent.CreateStart(100, "A1"),
StreamEvent.CreateEnd(101, 100, "A1"),
StreamEvent.CreateStart(103, "A2"),
StreamEvent.CreateEnd(105, 103, "A2"),
StreamEvent.CreateStart(103, "A1"),
StreamEvent.CreateEnd(106, 103, "A1"),
StreamEvent.CreateStart(109, "A1"),
StreamEvent.CreateEnd(110, 109, "A1"),
StreamEvent.CreateStart(102, "B1"),
StreamEvent.CreateEnd(112, 102, "B1"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(outputStream.IsEquivalentTo(correct));
}
private void CheckMessageType(string data)
{
try
{
Tweet tweet = JsonUtility.FromJson <Tweet>(data);
if (tweet.text != null && tweet.id_str != null)
{
messageType = StreamMessageType.Tweet;
return;
}
StreamEvent streamEvent = JsonUtility.FromJson <StreamEvent>(data);
if (streamEvent.event_name != null)
{
messageType = StreamMessageType.StreamEvent;
return;
}
FriendsList friendsList = JsonUtility.FromJson <FriendsList>(data);
if (friendsList.friends != null)
{
messageType = StreamMessageType.FriendsList;
return;
}
DirectMessage directMessage = JsonUtility.FromJson <DirectMessage>(data);
if (directMessage.recipient_screen_name != null)
{
messageType = StreamMessageType.DirectMessage;
return;
}
StatusDeletionNotice statusDeletionNotice = JsonUtility.FromJson <StatusDeletionNotice>(data);
if (statusDeletionNotice.delete != null)
{
messageType = StreamMessageType.StatusDeletionNotice;
return;
}
LocationDeletionNotice locationDeletionNotice = JsonUtility.FromJson <LocationDeletionNotice>(data);
if (locationDeletionNotice.scrub_geo != null)
{
messageType = StreamMessageType.LocationDeletionNotice;
return;
}
LimitNotice limitNotice = JsonUtility.FromJson <LimitNotice>(data);
if (limitNotice.limit != null)
{
messageType = StreamMessageType.LimitNotice;
return;
}
WithheldContentNotice withheldContentNotice = JsonUtility.FromJson <WithheldContentNotice>(data);
if (withheldContentNotice.status_withheld != null || withheldContentNotice.user_withheld != null)
{
messageType = StreamMessageType.WithheldContentNotice;
return;
}
DisconnectMessage disconnectMessage = JsonUtility.FromJson <DisconnectMessage>(data);
if (disconnectMessage.disconnect != null)
{
messageType = StreamMessageType.DisconnectMessage;
return;
}
StallWarning stallWarning = JsonUtility.FromJson <StallWarning>(data);
if (stallWarning.warning != null)
{
messageType = StreamMessageType.StallWarning;
return;
}
messageType = StreamMessageType.None;
return;
} catch (System.Exception e)
{
Debug.Log("CheckMessageType Error : " + e.ToString());
messageType = StreamMessageType.None;
return;
}
}
/// <inheritdoc/>
protected override async Task <IStreamProcessorState> OnSuccessfulProcessingResult(SuccessfulProcessing successfulProcessing, StreamEvent processedEvent, IStreamProcessorState currentState)
{
var newState = new StreamProcessorState(processedEvent.Position + 1, DateTimeOffset.UtcNow);
await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
return(newState);
}
object?Deserialize(StreamEvent streamEvent) => _serializer.Deserialize(streamEvent.Data.AsSpan(), streamEvent.EventType);
private void ProduceEvents()
{
try
{
using (var streamReader = this._twitterStreaming.Read())
{
// Loop until stop signal
while (!_disposed)
{
try
{
// Read from source
var line = streamReader.ReadLine();
if (string.IsNullOrEmpty(line))
{
continue;
}
// Parse
var jObject = JObject.Parse(line);
string text = Unquote(jObject.SelectToken("text").Value <string>());
string topic = TextAnalysis.DetermineTopc(text, _config.twitter_keywords);
string language = (null == jObject.SelectToken("user.lang")) ? string.Empty : jObject.SelectToken("user.lang").Value <string>();
// filter out tweets we don't care about: non-english tweets or deleted tweets
if ((jObject["delete"] != null) ||
(language != "en") ||
(topic == "Unknown"))
{
continue;
}
var tweet = new Tweet();
tweet.ID = jObject.SelectToken("id_str").Value <Int64>();
tweet.CreatedAt = ParseTwitterDateTime(jObject.SelectToken("created_at").Value <string>());
tweet.UserName = Unquote(jObject.SelectToken("user.screen_name").Value <string>());
tweet.ProfileImageUrl = jObject.SelectToken("user.profile_image_url").Value <string>();
tweet.Text = text;
tweet.Topic = topic;
tweet.RawJson = line;
tweet.SentimentScore = this.Sentiment(text);
// Produce INSERT event
Observer.OnNext(StreamEvent.CreatePoint(tweet.CreatedAt.Ticks, tweet));
}
catch (Exception ex)
{
// Error handling should go here
Console.WriteLine("EXCEPTION RAISED in TwitterInputAdapter: " + ex.Message);
}
}
}
}
catch (Exception e)
{
Observer.OnError(e);
TwitterStream.Dispose(_config);
}
}
/// <inheritdoc/>
protected override async Task <IStreamProcessorState> OnRetryProcessingResult(FailedProcessing failedProcessing, StreamEvent processedEvent, IStreamProcessorState currentState)
{
var oldState = currentState as StreamProcessorState;
var newState = new StreamProcessorState(
oldState.Position,
failedProcessing.FailureReason,
DateTimeOffset.UtcNow.Add(failedProcessing.RetryTimeout),
oldState.ProcessingAttempts + 1,
oldState.LastSuccessfullyProcessed,
true);
await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
return(newState);
}
static unsafe void CopyToBuffer(Span <byte> buffer, StreamEvent evt)
{
var length = (int)evt.Data.Recv.Buffers[0].Length;
new Span <byte>(evt.Data.Recv.Buffers[0].Buffer, length).CopyTo(buffer);
}
private static string BuildStreamName(StreamEvent arg)
{
var eventType = arg.Event.Type;
return($"et-{eventType}");
}
public void OnInstanceStreamEvent(SyncedData <StreamInstance> streamInstance, StreamEvent eventType)
{
switch (eventType)
{
case StreamEvent.Added:
m_InstanceCountData.addedCount++;
break;
case StreamEvent.Changed:
m_InstanceCountData.changedCount++;
break;
case StreamEvent.Removed:
m_InstanceCountData.removedCount++;
break;
}
m_Settings.instanceCountModified?.Invoke(m_InstanceCountData);
}
public void OnStreamEvent(object stream, StreamEvent eventType)
{
streamEvent?.Invoke(stream is T s ? s : default, eventType);
public void SendStreamEvent(T stream, StreamEvent eventType)
{
m_Inputs.ForEach(i => i.OnStreamEvent(stream, eventType));
}
public void GroupKeyLifetimeAccessTest()
{
var inputEvents = new StreamEvent <int>[]
{
StreamEvent.CreateStart(100, 1),
StreamEvent.CreateStart(200, 2),
StreamEvent.CreateStart(300, 3),
StreamEvent.CreateStart(400, 4),
StreamEvent.CreatePunctuation <int>(400),
StreamEvent.CreateStart(500, 5),
StreamEvent.CreateStart(600, 6),
StreamEvent.CreateStart(700, 7),
StreamEvent.CreateStart(800, 8),
StreamEvent.CreatePunctuation <int>(800),
StreamEvent.CreateStart(900, 9),
StreamEvent.CreateStart(1000, 10),
StreamEvent.CreateStart(1100, 11),
StreamEvent.CreateStart(1200, 12),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
};
var keyFactory = new TestKeyFactory();
var subject = new Subject <StreamEvent <int> >();
var output = subject
.ToStreamable()
.HoppingWindowLifetime(300, 100)
.GroupApply(
v => keyFactory.Create(v),
w => w.Count(),
(key, count) => new { key.Key, count });
var outputCounts = new List <ValueTuple <int, ulong> >();
var subscription = output.ToStreamEventObservable().Subscribe(onNext: item =>
{
if (item.IsData)
{
outputCounts.Add((item.Payload.Key.Key, item.Payload.count));
}
});
foreach (var inputEvent in inputEvents.Take(5))
{
subject.OnNext(inputEvent);
}
var oldKeys = keyFactory.KeysCreated.ToArray();
foreach (var inputEvent in inputEvents.Skip(5).Take(5))
{
subject.OnNext(inputEvent);
}
TestKeyFactory.MarkDirty(oldKeys);
foreach (var inputEvent in inputEvents.Skip(10))
{
subject.OnNext(inputEvent);
}
// Asserts are done while accessing records in TestKey methods
subject.OnCompleted();
subject.Dispose();
subscription.Dispose();
}
public static void Main(string[] args)
{
// Affinitize thread to core 0
NativeMethods.AffinitizeThread(0);
// Load a sample dataset into main memory
Console.WriteLine("Caching the input dataset into main memory...");
var sw0 = new Stopwatch();
sw0.Start();
var tumblingWindowDataset =
Observable.Range(0, TotalInputEvents)
.Select(e => StreamEvent.CreateInterval(((long)e / NumEventsPerTumble) * NumEventsPerTumble, ((long)e / NumEventsPerTumble) * NumEventsPerTumble + NumEventsPerTumble, new Payload {
field1 = 0, field2 = 0
}))
.ToStreamable()
.SetProperty().IsConstantDuration(true, NumEventsPerTumble)
.SetProperty().IsSnapshotSorted(true, e => e.field1)
.Cache();
var startEdgeOnlyDataset = tumblingWindowDataset
.AlterEventDuration(StreamEvent.InfinitySyncTime)
.SetProperty().IsSnapshotSorted(true, e => e.field1)
.Cache();
var singleSnapshotDataset = tumblingWindowDataset
.AlterEventLifetime(a => StreamEvent.MinSyncTime, StreamEvent.InfinitySyncTime)
.SetProperty().IsSnapshotSorted(true, e => e.field1)
.Cache();
sw0.Stop();
Console.WriteLine("Time to load cache with {0} tuples: {1} sec", TotalInputEvents, sw0.Elapsed);
Console.WriteLine("Press <ENTER> to continue");
Console.ReadLine();
Console.WriteLine("\n**** Queries over dataset with tumbling windows ****");
ProcessQuery(tumblingWindowDataset.Where(e => e.field1 != 0), "input.Where(e => e.field1 != 0)");
ProcessQuery(tumblingWindowDataset.Select(e => e.field1), "input.Select(e => e.field1)");
ProcessQuery(tumblingWindowDataset.Count(), "input.Count()");
ProcessQuery(tumblingWindowDataset.Sum(e => e.field2), "input.Sum(e => e.field2)");
ProcessQuery(tumblingWindowDataset.Average(e => e.field1), "input.Average(e => e.field1)");
ProcessQuery(tumblingWindowDataset.Max(e => e.field1), "input.Max(e => e.field1)");
ProcessQuery(tumblingWindowDataset.Min(e => e.field1), "input.Min(e => e.field1)");
ProcessQuery(tumblingWindowDataset
.GroupApply(
e => e.field2,
str => str.Count(),
(g, c) => new { Key = g, Count = c }),
"input.GroupApply(e => e.field2, str => str.Count(), (g, c) => new { Key = g, Count = c })");
Console.WriteLine("\n**** Queries over start-edge-only dataset ****");
ProcessQuery(startEdgeOnlyDataset.Count(), "input.Count()");
ProcessQuery(startEdgeOnlyDataset.Sum(e => e.field2), "input.Sum(e => e.field2)");
ProcessQuery(startEdgeOnlyDataset.Average(e => e.field1), "input.Average(e => e.field1)");
ProcessQuery(startEdgeOnlyDataset.Max(e => e.field1), "input.Max(e => e.field1)");
ProcessQuery(startEdgeOnlyDataset.Min(e => e.field1), "input.Min(e => e.field1)");
Console.WriteLine("\n**** Queries over a single-snapshot (atemporal) dataset ****");
ProcessQuery(singleSnapshotDataset.Count(), "input.Count()");
ProcessQuery(singleSnapshotDataset.Sum(e => e.field2), "input.Sum(e => e.field2)");
ProcessQuery(singleSnapshotDataset.Average(e => e.field1), "input.Average(e => e.field1)");
ProcessQuery(singleSnapshotDataset.Max(e => e.field1), "input.Max(e => e.field1)");
ProcessQuery(singleSnapshotDataset.Min(e => e.field1), "input.Min(e => e.field1)");
/* Dispose off the caches */
tumblingWindowDataset.Dispose();
startEdgeOnlyDataset.Dispose();
singleSnapshotDataset.Dispose();
Console.WriteLine("Press <ENTER>");
Console.ReadLine();
}
