/// <summary>
/// Build a DataFlows
/// </summary>
/// <returns>list of IDataflowObject for SdmxObject</returns>
public List <IDataflowObject> GetDataFlows()
{
if (ReferencesObject == null)
{
ReferencesObject = new IReferencesObject();
}
ReferencesObject.FoundedDataflows = new List <IDataflowObject>();
string DataflowCodeMaintenableId = this.parsingObject.MaintainableId;
MSDataflows = GetDataflows();
for (int i = MSDataflows.Count - 1; i >= 0; i--)
{
MSDataflow item = MSDataflows[i];
if ((!string.IsNullOrEmpty(DataflowCodeMaintenableId)) && item.DFCode.Trim().ToUpper() != DataflowCodeMaintenableId.Trim().ToUpper())
{
MSDataflows.RemoveAt(i);
continue;
}
DataflowBuilder dfbuilder = dfbuilder = new DataflowBuilder(item.DFCode, item.Descr, this.parsingObject, this.versionTypeResp);
//DsdManager dsdMan = new DsdManager(this.parsingObject.CloneForReferences(), this.versionTypeResp);
dfbuilder.DataStrunctureRef = item.GetDataStrunctureRef(MSDataflows[i].DsdCode, MSDataflows[i].DsdAgency, MSDataflows[i].DsdVersion);
ReferencesObject.FoundedDataflows.Add(dfbuilder.BuildDataflow(item.DFAgency, item.DFVersion));
}
return(ReferencesObject.FoundedDataflows);
}
public void TestMethod()
{
var source = new Source();
var intQueue = new ItemsQueueContainer <int>();
var doubleQueue = new ItemsQueueContainer <double>();
var exceptionHandler = Mock.Create <IExceptionHandler>();
var myWork = new MyWork();
var counter = new MyCounter();
var dataflow = DataflowBuilder
.StartFrom(source)
.TransmitWith(o => o.Custom(new MyTransmitterFactory()))
.SplitBatch()
.BuildTransformation(o => o.Transform(int.Parse)
.Transform(x => x.ToString())
.Transform(myWork))
.SendToPipe <int>(new InfinityPipe <int>(intQueue))
.TransmitWith(o => o.BalancedTransmitter(exceptionHandler, 0, 10)
.SetCounter(counters => counters.InWorkItemsCount = counter))
.RouteTo(condition => condition.Case(o => o % 3 == 0)
.Transform(o => o.ToString())
.SendToTarget(new LoggingTarget <string>("0) ", 100)),
condition => condition.Case(o => o % 3 == 1)
.BroadcastTo(target2 => target2.SendToTarget(new LoggingTarget <int>("1) ", 200)),
target2 => target2.ToBatches(4, TimeSpan.FromMilliseconds(500), exceptionHandler)
.SplitBatch()
.Transform(o => (double)o)
.Transform(o => (int)o)
.Transform(o => (double)o)
.SendToPipe <double>(new InfinityPipe <double>(doubleQueue))
.TransmitWith(o => o.BalancedTransmitter(exceptionHandler, 0, 10))
.BuildTransformation(o => o.ToBatches(4, TimeSpan.FromMilliseconds(500), exceptionHandler)
.SplitBatch())
.SendToTarget(new LoggingTarget <double>("2) "))),
condition => condition.Default()
.SendToTarget(new LoggingTarget <int>("3) ")))
.Build();
dataflow.Start();
Console.WriteLine("Started!");
Task.Delay(4000).Wait();
Console.WriteLine("I={0}, J={1}", counter.I, counter.J);
}
public void TestMethod2()
{
var exceptionHandler = Mock.Create <IExceptionHandler>();
Mock.Arrange(() => exceptionHandler.Handle(null)).IgnoreArguments().Returns <Exception>(o =>
{
Console.WriteLine(o);
return(true);
});
var itemStore = new ItemStore();
var uniqueQueue = new ItemsQueueContainer <ItemWithDate>();
var infinityPipe = new InfinityPipe <ItemWithDate>(uniqueQueue);
var dataflow = DataflowBuilder
.StartFrom(infinityPipe)
.ConvertItemsTo(o => new ScheduledTransmitterItem <int>(o.DateTime, o.Num, stopSchedule: o.Count == 4))
.TransmitWith(o => o.ScheduledTransmitter(new DateTimeProvider(), exceptionHandler))
.Transform(itemStore.GetItem)
.BroadcastTo(t1 => t1.SendToTarget(new LoggingTarget <ItemWithDate>()),
t2 => t2.Transform(o =>
{
o.Count++;
o.DateTime = DateTime.Now.AddSeconds(o.Num + 1);
return(o);
})
.RouteTo(condition => condition.Case(o => o.Count == 4)
.BroadcastTo(t21 => t21.SendToTarget(new DeleteTarget(itemStore)),
t22 => t22.SendToTarget(infinityPipe)),
condition => condition.Default()
.BroadcastTo(t21 => t21.SendToTarget(new UpdateTarget(itemStore)),
t22 => t22.SendToTarget(infinityPipe))))
.Build();
dataflow.Start();
var items = itemStore.Load();
foreach (var item in items)
{
infinityPipe.SendAsync(item).Wait();
}
Task.Delay(10000).Wait();
}
public bool TestWithInMemData()
{
CancellationTokenSource cts = new CancellationTokenSource();
// Dummy Snapshot storage
String _snapshot = string.Empty;
ConcurrentBag <String> snapshotbag = new ConcurrentBag <String>();
String[] messageArray = new string[1000];
// Populate dummy data
for (int x = 0; x < 1000; x++)
{
TestInput ti = new TestInput()
{
MessageID = x.ToString(), PayloadID = x.ToString(), Value = 1
};
messageArray[x] = JsonConvert.SerializeObject(ti);
}
// Dummy State Storage
TestState state = new TestState();
// Let's build a flow
DataflowBuilder <TestInput, TestState> dfBuilder =
new DataflowBuilder <TestInput, TestState>();
dfBuilder.Initialize <SyncDataflow <TestInput, TestState> >(). // Type of Flow is SyncDataflow
SetCancellationToken(cts.Token). // Token to be used to cancel flow processing
SetStartupBehaviour(enStartupBehaviour.StartAtLastSnapshot). // Start processing where we left off
SetSnapshotBehaviour(enSnapshotUnits.Count, 10). // Snapshot every 60 seconds
SetInputGetter((Snapshot <TestState> snap) => // Get Message from Array
{
int lastID = int.Parse(snap.LastMessageID); // Last Message id processed
// Signal end of input data
if (lastID >= messageArray.Length - 2) // Reacged end of Array - Signal end of flow
{
cts.Cancel();
}
String msg = messageArray[lastID + 1]; // Get Next Message
return(JsonConvert.DeserializeObject <TestInput>(msg));
}).
SetProcessor((Tuple <TestInput, TestState> i) => // Process Message
{
return(new TestState()
{
TotalValue = i.Item2.TotalValue + i.Item1.Value //Simple, just total it up
});
}).
SetSnapshotRetriever(() =>
{
return(JsonConvert.DeserializeObject <Snapshot <TestState> >(_snapshot)); // Use dummy snapshot
}).
SetSnapshotUpdater((Snapshot <TestState> snap) =>
{
_snapshot = JsonConvert.SerializeObject(snap); // Save snaphsot into dummy
System.Diagnostics.Debug.WriteLine($"Snap:{snap.LastMessageID}, {snap.State.TotalValue}");
}).
SetStatePublisher(s =>
{
System.Diagnostics.Debug.WriteLine($"Total:{s.TotalValue}");
}).
SetStateRetriever(() => state). // Get State from Dummy
SetStateInitializer(() => new TestState()
{
TotalValue = 0
}). // Initialize State if nothing is retrieved
SetSnapshotInitializer(() => new Snapshot <TestState>()
{
LastMessageID = "-1", LastTimeStamp = DateTimeOffset.MinValue.ToUnixTimeMilliseconds()
}). // Initialize Snapshot if nothing is retrieved
SetStateSetter(f =>
{
state.TotalValue = f.TotalValue; // Update State
}).Build().Start();
return(true);
}
public void ReadFromRedisStream()
{
// The Token allows for external control of the dataflow so that it could be stopped as needed
// We can use this to signal completion
CancellationTokenSource cts = new CancellationTokenSource();
// Must have Redis running locally, if not, change connection string to correct host
var redisDB = ConnectionMultiplexer.Connect("localhost").GetDatabase();
// Delete Stream from prior runs
redisDB.KeyDelete("tstData");
// Populate dummy data
for (int x = 0; x < 10000; x++)
{
TestInput ti = new TestInput()
{
PayloadID = x.ToString(), Value = 1
};
// Not providing an id means Redis will automatically assign an id which is returned by the StreamAdd command
// The StreamAdd also accepts a message id to allow the producer to control the messageid
// But, Redis expects the streamids to continuously increment
redisDB.StreamAdd("tstData", "data", JsonConvert.SerializeObject(ti));
}
DataflowBuilder <TestInput, TestState> dfBuilder =
new DataflowBuilder <TestInput, TestState>();
dfBuilder.Initialize <SyncDataflow <TestInput, TestState> >(). // Use SyncDataflow as execution engine
SetCancellationToken(cts.Token). // Use this to signal end of stream
SetStartupBehaviour(enStartupBehaviour.StartAtLastSnapshot). // start from last snapshot
SetSnapshotBehaviour(enSnapshotUnits.Count, 100). // Save snapshot every 10 items
SetInputGetter((Snapshot <TestState> snap) => // Get next item
{
StreamEntry[] entries = redisDB.StreamRead("tstData", snap.LastMessageID, 1);
if ((entries.Length == 1) && (entries[0].Values.Length == 1))
{
TestInput input = JsonConvert.DeserializeObject <TestInput>(entries[0].Values[0].Value);
// If Redis ID was autogenerated it would differ from internal ID so override
// May have to Work on that a bit
if (input.MessageID != entries[0].Id)
{
input.MessageID = entries[0].Id;
}
return(input);
}
else
{
cts.Cancel(); // Got null = end of stream - Signal to the engine to end.
return(null); // In a real application you would not stop at end of stream, you would keep polling
}
}).
SetProcessor((Tuple <TestInput, TestState> i) => // Process Message
{
TestState result = new TestState()
{
TotalValue = i.Item2.TotalValue + i.Item1.Value //Simple, just total it up
};
return(result);
}).
SetSnapshotRetriever(() => // Get last snapshot
{
string ss = redisDB.StringGet("tstDataSnapshot");
if (!String.IsNullOrEmpty(ss))
{
return(JsonConvert.DeserializeObject <Snapshot <TestState> >(redisDB.StringGet("tstDataSnapshot")));
}
else
{
return(null);
}
}).
SetSnapshotUpdater((Snapshot <TestState> snap) =>
{
redisDB.StringSet("tstDataSnapshot", JsonConvert.SerializeObject(snap));
System.Diagnostics.Debug.WriteLine($"Snap:{snap.LastMessageID}, {snap.State.TotalValue}");
}).
SetSnapshotInitializer(() => new Snapshot <TestState>()
{
LastMessageID = "0-0", LastTimeStamp = DateTimeOffset.MinValue.ToUnixTimeMilliseconds()
}). // Initialize Snapshot if nothing is retrieved
SetStateInitializer(() => new TestState()
{
TotalValue = 0
}). // Initialize State if nothing is retrieved
SetStatePublisher(s =>
{
System.Diagnostics.Debug.WriteLine($"Total:{s.TotalValue}");
}).
SetStateRetriever(() =>
{
string ss = redisDB.StringGet("tstDataSnapshot");
if (!String.IsNullOrEmpty(ss))
{
return(JsonConvert.DeserializeObject <TestState>(redisDB.StringGet("tstDataState")));
}
else
{
return(null);
}
}).
SetStateSetter(f =>
{
redisDB.StringSet("tstDataState", JsonConvert.SerializeObject(f));
}).Build().Start();
}
