public bool TestWithInMemData()
{
CancellationTokenSource cts = new CancellationTokenSource();
// Dummy Snapshot storage
String _snapshot = string.Empty;
ConcurrentBag <String> snapshotbag = new ConcurrentBag <String>();
String[] messageArray = new string[1000];
// Populate dummy data
for (int x = 0; x < 1000; x++)
{
TestInput ti = new TestInput()
{
MessageID = x.ToString(), PayloadID = x.ToString(), Value = 1
};
messageArray[x] = JsonConvert.SerializeObject(ti);
}
// Dummy State Storage
TestState state = new TestState();
// Let's build a flow
DataflowBuilder <TestInput, TestState> dfBuilder =
new DataflowBuilder <TestInput, TestState>();
dfBuilder.Initialize <SyncDataflow <TestInput, TestState> >(). // Type of Flow is SyncDataflow
SetCancellationToken(cts.Token). // Token to be used to cancel flow processing
SetStartupBehaviour(enStartupBehaviour.StartAtLastSnapshot). // Start processing where we left off
SetSnapshotBehaviour(enSnapshotUnits.Count, 10). // Snapshot every 60 seconds
SetInputGetter((Snapshot <TestState> snap) => // Get Message from Array
{
int lastID = int.Parse(snap.LastMessageID); // Last Message id processed
// Signal end of input data
if (lastID >= messageArray.Length - 2) // Reacged end of Array - Signal end of flow
{
cts.Cancel();
}
String msg = messageArray[lastID + 1]; // Get Next Message
return(JsonConvert.DeserializeObject <TestInput>(msg));
}).
SetProcessor((Tuple <TestInput, TestState> i) => // Process Message
{
return(new TestState()
{
TotalValue = i.Item2.TotalValue + i.Item1.Value //Simple, just total it up
});
}).
SetSnapshotRetriever(() =>
{
return(JsonConvert.DeserializeObject <Snapshot <TestState> >(_snapshot)); // Use dummy snapshot
}).
SetSnapshotUpdater((Snapshot <TestState> snap) =>
{
_snapshot = JsonConvert.SerializeObject(snap); // Save snaphsot into dummy
System.Diagnostics.Debug.WriteLine($"Snap:{snap.LastMessageID}, {snap.State.TotalValue}");
}).
SetStatePublisher(s =>
{
System.Diagnostics.Debug.WriteLine($"Total:{s.TotalValue}");
}).
SetStateRetriever(() => state). // Get State from Dummy
SetStateInitializer(() => new TestState()
{
TotalValue = 0
}). // Initialize State if nothing is retrieved
SetSnapshotInitializer(() => new Snapshot <TestState>()
{
LastMessageID = "-1", LastTimeStamp = DateTimeOffset.MinValue.ToUnixTimeMilliseconds()
}). // Initialize Snapshot if nothing is retrieved
SetStateSetter(f =>
{
state.TotalValue = f.TotalValue; // Update State
}).Build().Start();
return(true);
}
public void ReadFromRedisStream()
{
// The Token allows for external control of the dataflow so that it could be stopped as needed
// We can use this to signal completion
CancellationTokenSource cts = new CancellationTokenSource();
// Must have Redis running locally, if not, change connection string to correct host
var redisDB = ConnectionMultiplexer.Connect("localhost").GetDatabase();
// Delete Stream from prior runs
redisDB.KeyDelete("tstData");
// Populate dummy data
for (int x = 0; x < 10000; x++)
{
TestInput ti = new TestInput()
{
PayloadID = x.ToString(), Value = 1
};
// Not providing an id means Redis will automatically assign an id which is returned by the StreamAdd command
// The StreamAdd also accepts a message id to allow the producer to control the messageid
// But, Redis expects the streamids to continuously increment
redisDB.StreamAdd("tstData", "data", JsonConvert.SerializeObject(ti));
}
DataflowBuilder <TestInput, TestState> dfBuilder =
new DataflowBuilder <TestInput, TestState>();
dfBuilder.Initialize <SyncDataflow <TestInput, TestState> >(). // Use SyncDataflow as execution engine
SetCancellationToken(cts.Token). // Use this to signal end of stream
SetStartupBehaviour(enStartupBehaviour.StartAtLastSnapshot). // start from last snapshot
SetSnapshotBehaviour(enSnapshotUnits.Count, 100). // Save snapshot every 10 items
SetInputGetter((Snapshot <TestState> snap) => // Get next item
{
StreamEntry[] entries = redisDB.StreamRead("tstData", snap.LastMessageID, 1);
if ((entries.Length == 1) && (entries[0].Values.Length == 1))
{
TestInput input = JsonConvert.DeserializeObject <TestInput>(entries[0].Values[0].Value);
// If Redis ID was autogenerated it would differ from internal ID so override
// May have to Work on that a bit
if (input.MessageID != entries[0].Id)
{
input.MessageID = entries[0].Id;
}
return(input);
}
else
{
cts.Cancel(); // Got null = end of stream - Signal to the engine to end.
return(null); // In a real application you would not stop at end of stream, you would keep polling
}
}).
SetProcessor((Tuple <TestInput, TestState> i) => // Process Message
{
TestState result = new TestState()
{
TotalValue = i.Item2.TotalValue + i.Item1.Value //Simple, just total it up
};
return(result);
}).
SetSnapshotRetriever(() => // Get last snapshot
{
string ss = redisDB.StringGet("tstDataSnapshot");
if (!String.IsNullOrEmpty(ss))
{
return(JsonConvert.DeserializeObject <Snapshot <TestState> >(redisDB.StringGet("tstDataSnapshot")));
}
else
{
return(null);
}
}).
SetSnapshotUpdater((Snapshot <TestState> snap) =>
{
redisDB.StringSet("tstDataSnapshot", JsonConvert.SerializeObject(snap));
System.Diagnostics.Debug.WriteLine($"Snap:{snap.LastMessageID}, {snap.State.TotalValue}");
}).
SetSnapshotInitializer(() => new Snapshot <TestState>()
{
LastMessageID = "0-0", LastTimeStamp = DateTimeOffset.MinValue.ToUnixTimeMilliseconds()
}). // Initialize Snapshot if nothing is retrieved
SetStateInitializer(() => new TestState()
{
TotalValue = 0
}). // Initialize State if nothing is retrieved
SetStatePublisher(s =>
{
System.Diagnostics.Debug.WriteLine($"Total:{s.TotalValue}");
}).
SetStateRetriever(() =>
{
string ss = redisDB.StringGet("tstDataSnapshot");
if (!String.IsNullOrEmpty(ss))
{
return(JsonConvert.DeserializeObject <TestState>(redisDB.StringGet("tstDataState")));
}
else
{
return(null);
}
}).
SetStateSetter(f =>
{
redisDB.StringSet("tstDataState", JsonConvert.SerializeObject(f));
}).Build().Start();
}