/// <summary> /// Build a DataFlows /// </summary> /// <returns>list of IDataflowObject for SdmxObject</returns> public List <IDataflowObject> GetDataFlows() { if (ReferencesObject == null) { ReferencesObject = new IReferencesObject(); } ReferencesObject.FoundedDataflows = new List <IDataflowObject>(); string DataflowCodeMaintenableId = this.parsingObject.MaintainableId; MSDataflows = GetDataflows(); for (int i = MSDataflows.Count - 1; i >= 0; i--) { MSDataflow item = MSDataflows[i]; if ((!string.IsNullOrEmpty(DataflowCodeMaintenableId)) && item.DFCode.Trim().ToUpper() != DataflowCodeMaintenableId.Trim().ToUpper()) { MSDataflows.RemoveAt(i); continue; } DataflowBuilder dfbuilder = dfbuilder = new DataflowBuilder(item.DFCode, item.Descr, this.parsingObject, this.versionTypeResp); //DsdManager dsdMan = new DsdManager(this.parsingObject.CloneForReferences(), this.versionTypeResp); dfbuilder.DataStrunctureRef = item.GetDataStrunctureRef(MSDataflows[i].DsdCode, MSDataflows[i].DsdAgency, MSDataflows[i].DsdVersion); ReferencesObject.FoundedDataflows.Add(dfbuilder.BuildDataflow(item.DFAgency, item.DFVersion)); } return(ReferencesObject.FoundedDataflows); }
public void TestMethod() { var source = new Source(); var intQueue = new ItemsQueueContainer <int>(); var doubleQueue = new ItemsQueueContainer <double>(); var exceptionHandler = Mock.Create <IExceptionHandler>(); var myWork = new MyWork(); var counter = new MyCounter(); var dataflow = DataflowBuilder .StartFrom(source) .TransmitWith(o => o.Custom(new MyTransmitterFactory())) .SplitBatch() .BuildTransformation(o => o.Transform(int.Parse) .Transform(x => x.ToString()) .Transform(myWork)) .SendToPipe <int>(new InfinityPipe <int>(intQueue)) .TransmitWith(o => o.BalancedTransmitter(exceptionHandler, 0, 10) .SetCounter(counters => counters.InWorkItemsCount = counter)) .RouteTo(condition => condition.Case(o => o % 3 == 0) .Transform(o => o.ToString()) .SendToTarget(new LoggingTarget <string>("0) ", 100)), condition => condition.Case(o => o % 3 == 1) .BroadcastTo(target2 => target2.SendToTarget(new LoggingTarget <int>("1) ", 200)), target2 => target2.ToBatches(4, TimeSpan.FromMilliseconds(500), exceptionHandler) .SplitBatch() .Transform(o => (double)o) .Transform(o => (int)o) .Transform(o => (double)o) .SendToPipe <double>(new InfinityPipe <double>(doubleQueue)) .TransmitWith(o => o.BalancedTransmitter(exceptionHandler, 0, 10)) .BuildTransformation(o => o.ToBatches(4, TimeSpan.FromMilliseconds(500), exceptionHandler) .SplitBatch()) .SendToTarget(new LoggingTarget <double>("2) "))), condition => condition.Default() .SendToTarget(new LoggingTarget <int>("3) "))) .Build(); dataflow.Start(); Console.WriteLine("Started!"); Task.Delay(4000).Wait(); Console.WriteLine("I={0}, J={1}", counter.I, counter.J); }
public void TestMethod2() { var exceptionHandler = Mock.Create <IExceptionHandler>(); Mock.Arrange(() => exceptionHandler.Handle(null)).IgnoreArguments().Returns <Exception>(o => { Console.WriteLine(o); return(true); }); var itemStore = new ItemStore(); var uniqueQueue = new ItemsQueueContainer <ItemWithDate>(); var infinityPipe = new InfinityPipe <ItemWithDate>(uniqueQueue); var dataflow = DataflowBuilder .StartFrom(infinityPipe) .ConvertItemsTo(o => new ScheduledTransmitterItem <int>(o.DateTime, o.Num, stopSchedule: o.Count == 4)) .TransmitWith(o => o.ScheduledTransmitter(new DateTimeProvider(), exceptionHandler)) .Transform(itemStore.GetItem) .BroadcastTo(t1 => t1.SendToTarget(new LoggingTarget <ItemWithDate>()), t2 => t2.Transform(o => { o.Count++; o.DateTime = DateTime.Now.AddSeconds(o.Num + 1); return(o); }) .RouteTo(condition => condition.Case(o => o.Count == 4) .BroadcastTo(t21 => t21.SendToTarget(new DeleteTarget(itemStore)), t22 => t22.SendToTarget(infinityPipe)), condition => condition.Default() .BroadcastTo(t21 => t21.SendToTarget(new UpdateTarget(itemStore)), t22 => t22.SendToTarget(infinityPipe)))) .Build(); dataflow.Start(); var items = itemStore.Load(); foreach (var item in items) { infinityPipe.SendAsync(item).Wait(); } Task.Delay(10000).Wait(); }
public bool TestWithInMemData() { CancellationTokenSource cts = new CancellationTokenSource(); // Dummy Snapshot storage String _snapshot = string.Empty; ConcurrentBag <String> snapshotbag = new ConcurrentBag <String>(); String[] messageArray = new string[1000]; // Populate dummy data for (int x = 0; x < 1000; x++) { TestInput ti = new TestInput() { MessageID = x.ToString(), PayloadID = x.ToString(), Value = 1 }; messageArray[x] = JsonConvert.SerializeObject(ti); } // Dummy State Storage TestState state = new TestState(); // Let's build a flow DataflowBuilder <TestInput, TestState> dfBuilder = new DataflowBuilder <TestInput, TestState>(); dfBuilder.Initialize <SyncDataflow <TestInput, TestState> >(). // Type of Flow is SyncDataflow SetCancellationToken(cts.Token). // Token to be used to cancel flow processing SetStartupBehaviour(enStartupBehaviour.StartAtLastSnapshot). // Start processing where we left off SetSnapshotBehaviour(enSnapshotUnits.Count, 10). // Snapshot every 60 seconds SetInputGetter((Snapshot <TestState> snap) => // Get Message from Array { int lastID = int.Parse(snap.LastMessageID); // Last Message id processed // Signal end of input data if (lastID >= messageArray.Length - 2) // Reacged end of Array - Signal end of flow { cts.Cancel(); } String msg = messageArray[lastID + 1]; // Get Next Message return(JsonConvert.DeserializeObject <TestInput>(msg)); }). SetProcessor((Tuple <TestInput, TestState> i) => // Process Message { return(new TestState() { TotalValue = i.Item2.TotalValue + i.Item1.Value //Simple, just total it up }); }). SetSnapshotRetriever(() => { return(JsonConvert.DeserializeObject <Snapshot <TestState> >(_snapshot)); // Use dummy snapshot }). SetSnapshotUpdater((Snapshot <TestState> snap) => { _snapshot = JsonConvert.SerializeObject(snap); // Save snaphsot into dummy System.Diagnostics.Debug.WriteLine($"Snap:{snap.LastMessageID}, {snap.State.TotalValue}"); }). SetStatePublisher(s => { System.Diagnostics.Debug.WriteLine($"Total:{s.TotalValue}"); }). SetStateRetriever(() => state). // Get State from Dummy SetStateInitializer(() => new TestState() { TotalValue = 0 }). // Initialize State if nothing is retrieved SetSnapshotInitializer(() => new Snapshot <TestState>() { LastMessageID = "-1", LastTimeStamp = DateTimeOffset.MinValue.ToUnixTimeMilliseconds() }). // Initialize Snapshot if nothing is retrieved SetStateSetter(f => { state.TotalValue = f.TotalValue; // Update State }).Build().Start(); return(true); }
public void ReadFromRedisStream() { // The Token allows for external control of the dataflow so that it could be stopped as needed // We can use this to signal completion CancellationTokenSource cts = new CancellationTokenSource(); // Must have Redis running locally, if not, change connection string to correct host var redisDB = ConnectionMultiplexer.Connect("localhost").GetDatabase(); // Delete Stream from prior runs redisDB.KeyDelete("tstData"); // Populate dummy data for (int x = 0; x < 10000; x++) { TestInput ti = new TestInput() { PayloadID = x.ToString(), Value = 1 }; // Not providing an id means Redis will automatically assign an id which is returned by the StreamAdd command // The StreamAdd also accepts a message id to allow the producer to control the messageid // But, Redis expects the streamids to continuously increment redisDB.StreamAdd("tstData", "data", JsonConvert.SerializeObject(ti)); } DataflowBuilder <TestInput, TestState> dfBuilder = new DataflowBuilder <TestInput, TestState>(); dfBuilder.Initialize <SyncDataflow <TestInput, TestState> >(). // Use SyncDataflow as execution engine SetCancellationToken(cts.Token). // Use this to signal end of stream SetStartupBehaviour(enStartupBehaviour.StartAtLastSnapshot). // start from last snapshot SetSnapshotBehaviour(enSnapshotUnits.Count, 100). // Save snapshot every 10 items SetInputGetter((Snapshot <TestState> snap) => // Get next item { StreamEntry[] entries = redisDB.StreamRead("tstData", snap.LastMessageID, 1); if ((entries.Length == 1) && (entries[0].Values.Length == 1)) { TestInput input = JsonConvert.DeserializeObject <TestInput>(entries[0].Values[0].Value); // If Redis ID was autogenerated it would differ from internal ID so override // May have to Work on that a bit if (input.MessageID != entries[0].Id) { input.MessageID = entries[0].Id; } return(input); } else { cts.Cancel(); // Got null = end of stream - Signal to the engine to end. return(null); // In a real application you would not stop at end of stream, you would keep polling } }). SetProcessor((Tuple <TestInput, TestState> i) => // Process Message { TestState result = new TestState() { TotalValue = i.Item2.TotalValue + i.Item1.Value //Simple, just total it up }; return(result); }). SetSnapshotRetriever(() => // Get last snapshot { string ss = redisDB.StringGet("tstDataSnapshot"); if (!String.IsNullOrEmpty(ss)) { return(JsonConvert.DeserializeObject <Snapshot <TestState> >(redisDB.StringGet("tstDataSnapshot"))); } else { return(null); } }). SetSnapshotUpdater((Snapshot <TestState> snap) => { redisDB.StringSet("tstDataSnapshot", JsonConvert.SerializeObject(snap)); System.Diagnostics.Debug.WriteLine($"Snap:{snap.LastMessageID}, {snap.State.TotalValue}"); }). SetSnapshotInitializer(() => new Snapshot <TestState>() { LastMessageID = "0-0", LastTimeStamp = DateTimeOffset.MinValue.ToUnixTimeMilliseconds() }). // Initialize Snapshot if nothing is retrieved SetStateInitializer(() => new TestState() { TotalValue = 0 }). // Initialize State if nothing is retrieved SetStatePublisher(s => { System.Diagnostics.Debug.WriteLine($"Total:{s.TotalValue}"); }). SetStateRetriever(() => { string ss = redisDB.StringGet("tstDataSnapshot"); if (!String.IsNullOrEmpty(ss)) { return(JsonConvert.DeserializeObject <TestState>(redisDB.StringGet("tstDataState"))); } else { return(null); } }). SetStateSetter(f => { redisDB.StringSet("tstDataState", JsonConvert.SerializeObject(f)); }).Build().Start(); }