public void GetPowerBIDataflowDatasourceIndividualScope_ObjectAndListParameterSet() { // Arrange var testDataflow = new Dataflow { Id = Guid.NewGuid(), Name = "TestDataflow" }; var expectedDatasources = new List <Datasource> { new Datasource { DatasourceId = Guid.NewGuid().ToString(), Name = "TestDatasource", GatewayId = Guid.NewGuid().ToString() } }; var workspaceId = Guid.NewGuid(); var client = new Mock <IPowerBIApiClient>(); client.Setup(x => x.Dataflows.GetDataflowDatasources(workspaceId, testDataflow.Id)).Returns(expectedDatasources); var initFactory = new TestPowerBICmdletInitFactory(client.Object); var cmdlet = new GetPowerBIDataflowDatasource(initFactory) { Scope = PowerBIUserScope.Individual, Dataflow = testDataflow, WorkspaceId = workspaceId, ParameterSet = "DataflowAndList", }; // Act cmdlet.InvokePowerBICmdlet(); // Assert initFactory.AssertExpectedUnitTestResults(expectedDatasources); }
public RingMonitor(Dataflow host, IRingNode[] ringNodes) { if (ringNodes.Length == 0) { throw new ArgumentException("The child ring contains nothing"); } m_hb = ringNodes.OfType <IHeartbeatNode>().FirstOrDefault(); if (m_hb == null) { throw new ArgumentException("A ring must contain at least one IHeartbeatNode"); } var nonChild = ringNodes.FirstOrDefault(r => { return(!host.IsMyChild(r)); }); if (nonChild != null) { throw new ArgumentException("The child ring contains a non-child: " + nonChild.FullName); } this.m_host = host; this.m_ringNodes = ringNodes; m_lazyDisplayName = new Lazy <string>(this.GetDisplayName); }
public PipeLine( DataflowOptions dataflowOptions, int maxDegreeOfParallelism, HttpClient client) : base(dataflowOptions) { _client = client; _dataflowOptions = dataflowOptions; _options = dataflowOptions.ToExecutionBlockOption(true); _options.MaxDegreeOfParallelism = maxDegreeOfParallelism; _options.BoundedCapacity = 10000; _options.MaxMessagesPerTask = 100; _options.SingleProducerConstrained = true; _requestBlock = new TransformBlock <string, HttpResponseMessage>(x => { Interlocked.Increment(ref _totalOut); Console.WriteLine("lines " + _totalOut); var response = _client.GetAsync("http://localhost:3000/endpoint/info").Result; return(default(HttpResponseMessage)); }, _options); _flowBlock = new ActionBlock <HttpResponseMessage>(x => { }, _options); _request = _requestBlock.ToDataflow(_dataflowOptions, "Request"); _flow = _flowBlock.ToDataflow(_dataflowOptions, "Flow"); _request.LinkTo(_flow); }
public CrawlersFlow(IEnumerable <Func <IReadOnlyList <string>, IReadOnlyList <IData> > > crawlers) : base(DataflowOptions.Default) { if (crawlers is null) { throw new ArgumentNullException(nameof(crawlers)); } _inputBroadcaster = new DataBroadcaster <IReadOnlyList <string> >(filteredData => { Console.WriteLine($"Broadcasts all filtered inputters data. {filteredData.Count.ToString()}"); return(filteredData); }, DataflowOptions.Default); _resultConsumer = new TransformBlock <IReadOnlyList <IData>, IReadOnlyList <IData> >( crawlersData => crawlersData ).ToDataflow(DataflowOptions.Default); var crawlerFlows = crawlers .Select(crawler => DataflowUtils.FromDelegate(crawler, DataflowOptions.Default)); foreach (var crawlerFlow in crawlerFlows) { _inputBroadcaster.LinkTo(crawlerFlow); crawlerFlow.LinkTo(_resultConsumer); _resultConsumer.RegisterDependency(crawlerFlow); RegisterChild(crawlerFlow); } RegisterChild(_inputBroadcaster); RegisterChild(_resultConsumer); }
public OutputtersFlow( IEnumerable <Action <IReadOnlyList <string> > > outputters) : base(DataflowOptions.Default) { if (outputters is null) { throw new ArgumentNullException(nameof(outputters)); } _resultList = new ConcurrentBag <string>(); _inputConsumer = new DataBroadcaster <IReadOnlyList <string> >(appraisersData => { Console.WriteLine($"Consuming all appraised data. {appraisersData.Count.ToString()}\n"); foreach (string datum in appraisersData) { _resultList.Add(datum); } return(appraisersData); }, DataflowOptions.Default); var outputterFlows = outputters .Select(outputters => DataflowUtils.FromDelegate(outputters, DataflowOptions.Default)); foreach (var outputterFlow in outputterFlows) { _inputConsumer.LinkTo(outputterFlow); RegisterChild(outputterFlow); } RegisterChild(_inputConsumer); }
public void OpenDataflow(Dataflow d) { //Nodes List <DataflowNode> nodes = new List <DataflowNode>(); for (int n = 0; n < d.nodes.Count; n++) { DrawNode(d.nodes[n]); } //Links for (int n = 0; n < d.nodes.Count; n++) { DataflowNode node = d.nodes[n]; for (int i = 0; i < node.outputs.Count; i++) { for (int l = 0; l < node.outputs[i].links.Count; l++) { DataflowOutput output = node.outputs[i]; DataflowInput input = node.outputs[i].links[l]; UINodeLink uiNodeLink = Instantiate(Resources.Load <UINodeLink>("Link"), panel); uiNodeLink.outputAnchor = output.transform.GetChild(1).GetComponent <RectTransform>(); uiNodeLink.inputAnchor = input.transform.GetChild(1).GetComponent <RectTransform>(); input.uiLink = uiNodeLink; } } } dataflow = d; RefreshDataflowDropdown(); RefreshDroneDropdown(); }
public Dataflow ToDataflow() { Dataflow d = new Dataflow(name); foreach (var n in nodes) { DataflowNode node = d.AddNode(n.name, n.posx, n.posy); node.activator.valueFloat = n.activator.valueFloat; node.activator.valueBoolean = n.activator.valueBoolean; node.activator.ready = n.activator.ready; for (int i = 0; i < node.inputs.Count; i++) { node.inputs[i].valueFloat = n.inputs[i].valueFloat; node.inputs[i].valueBoolean = n.inputs[i].valueBoolean; node.inputs[i].ready = n.inputs[i].ready; } foreach (var c in n.configs) { node.configs[c.name].value = c.value; } } foreach (var l in links) { if (l.outputId < d.nodes[l.outputNodeId].outputs.Count && l.inputId < d.nodes[l.inputNodeId].inputs.Count) //Check for different save versions (inputs/outputs changed) { d.AddLink(d.nodes[l.outputNodeId].outputs[l.outputId], l.inputId < 0 ? d.nodes[l.inputNodeId].activator : d.nodes[l.inputNodeId].inputs[l.inputId]); } } return(d); }
public ImageSource() { source = new DataBroadcaster <BitmapSource>(DataflowOptions.Default) { Name = this.Name }; AddOutputConnector(new BitmapSourceOutputConnectorViewModel(this, "Output", Colors.DarkSeaGreen, source.OutputBlock)); }
public LineAggregatorFlow() : base(DataflowOptions.Default) { this._lineProcessor = new TransformManyBlock <string, string>(line => line.Split(' ')).ToDataflow(); this._aggregator = new AggregatorFlow(); this._lineProcessor.LinkTo(this._aggregator); this.RegisterChild(this._lineProcessor); this.RegisterChild(this._aggregator); }
public MyDataflow() : base(DataflowOptions.Default) { m_block1 = new BufferBlock <T>().ToDataflow(name: "innerflow1"); m_block2 = new BatchBlock <T>(100).ToDataflow(name: "innerflow2"); m_block1.LinkTo(m_block2); RegisterChild(this.m_block1); RegisterChild(this.m_block2); }
public void TestDuplicateRegistration() { var d1 = new Dataflow(DataflowOptions.Default); var d2 = new Dataflow(DataflowOptions.Default); d1.RegisterChild(d2); d1.RegisterChild(d2); }
public DataflowSave(Dataflow dataflow) { name = dataflow.name; foreach (var dn in dataflow.nodes) { Node node = new Node { name = dn.name, posx = dn.posx, posy = dn.posy, inputs = new List <Input>(), configs = new List <Config>() }; var dna = dn.activator; node.activator = new Input { valueFloat = dna.valueFloat, valueBoolean = dna.valueBoolean, ready = dna.ready }; if (dna.link != null) { links.Add(new Link { inputNodeId = dataflow.nodes.IndexOf(dn), inputId = -1, outputNodeId = dataflow.nodes.IndexOf(dna.link.node), outputId = dna.link.node.outputs.IndexOf(dna.link) }); } foreach (var dni in dn.inputs) { node.inputs.Add(new Input { valueFloat = dni.valueFloat, valueBoolean = dni.valueBoolean, ready = dni.ready }); if (dni.link != null) { links.Add(new Link { inputNodeId = dataflow.nodes.IndexOf(dn), inputId = dn.inputs.IndexOf(dni), outputNodeId = dataflow.nodes.IndexOf(dni.link.node), outputId = dni.link.node.outputs.IndexOf(dni.link) }); } } foreach (var dnc in dn.configs) { node.configs.Add(new Config { name = dnc.Key, value = dnc.Value.value }); } nodes.Add(node); } }
public void CloseDataflow() { for (int i = 0; i < panel.childCount; i++) { Destroy(panel.GetChild(i).gameObject); } dataflow = null; }
protected BitmapSourceElement() { source = new TransformBlock <BitmapSource, BitmapSource>(X => displayImage(X), new ExecutionDataflowBlockOptions() { TaskScheduler = TaskScheduler.FromCurrentSynchronizationContext() }) .ToDataflow(DataflowOptions.Default, Name); AddOutputConnector(new BitmapSourceOutputConnectorViewModel(this, "Output", Colors.DarkSeaGreen, source.OutputBlock)); }
static void dbStart() { try { _connect = new SqlConnection(_DB_CONST.get_connectString_Mobility()); //_connect.Open(); } catch (Exception ex) { Dataflow.enqueue(new JobLogPrintOut(ex.Message)).Wait(); } }
public override void Initialize(SsisEmitterContext context) { Component = Dataflow.NewComponentMetadata(); Component.ComponentClassID = Moniker; Instance = Component.Instantiate(); Instance.ProvideComponentProperties(); Component.Name = Name; Component.ValidateExternalMetadata = ValidateExternalMetadata; }
public void TestCircularDependency() { var d1 = new Dataflow(DataflowOptions.Default); var d2 = new Dataflow(DataflowOptions.Default); var block = new ActionBlock <int>(i => { }); d2.RegisterChild(block); d1.RegisterChild(d2); d2.RegisterChild(d1); }
public async Task AssertFlowError <T>(Dataflow f) where T : Exception { try { await f.CompletionTask; } catch (Exception e) { Assert.IsTrue(TaskEx.UnwrapWithPriority(e as AggregateException) is T); } }
public ProjectionBufferedDispatcher(DataflowOptions dataflowOptions, ProjectionBase <TState> projection) : base(dataflowOptions) { _buffer = new BufferBlock <Tracked <IStream> >(dataflowOptions.ToDataflowBlockOptions(false)).ToDataflow(dataflowOptions); _dispatcher = new ProjectionDispatcher <TState>(dataflowOptions, projection); _log = projection.Log; _token = projection.CancellationToken; _token.Register(() => _buffer.LinkTo(DataflowBlock.NullTarget <Tracked <IStream> >().ToDataflow(dataflowOptions))); RegisterChild(_buffer); }
static void fn_processClientCommand(string msg, WebSocket socketClient) { if (string.IsNullOrWhiteSpace(msg)) { return; } msg = msg.Trim(); Console.WriteLine("-> APP_CLIENT: " + msg); string url = "", keyCache = ""; string[] a; CLIENT_COMMAND cmd = CLIENT_COMMAND.NONE; ObjectCache cacheRuntime = MemoryCache.Default; int user_ID = 0; switch (msg[0]) { case '#': // #EXPORT.PDF:User_ID.hop_dong.Pawn_ID = #EXPORT.PDF:9.hop_dong.1167678 a = msg.Substring(1).Split(':'); switch (a[0]) { case "EXPORT.PDF": if (a.Length > 1) { a = a[1].Split('.'); if (a.Length > 2) { if (int.TryParse(a[0], out user_ID) && user_ID > 0) { keyCache = "APP_CLIENT_SOCKET." + a[0]; cacheRuntime.Set(keyCache, socketClient, new CacheItemPolicy()); cmd = CLIENT_COMMAND.EXPORT_PDF; //_dataflow.enqueue(new JobPdfExport("http://localhost:9096/api/pawn_info/get_in_hop_dong?Pawn_ID=1167678&filetemp=in-hop-dong.html")).Wait(); url = "http://localhost:9096/api/pawn_info/get_in_" + a[1] + "?User_ID=" + a[0] + "&Pawn_ID=" + a[2] + "&filetemp=in-" + a[1].Replace("_", "-") + ".html"; } } } break; } break; case '!': break; } switch (cmd) { case CLIENT_COMMAND.EXPORT_PDF: Dataflow.enqueue(new JobPdfExport(url)).Wait(); break; } }
public void Add <T>(Dataflow <TMessage, T> item) where T : class, IEventMetadata { if (typeof(T) != typeof(IEventMetadata)) { _eventBag.Add(item as Dataflow <TMessage, IEvent>); } else { _metadataBag.Add(item as Dataflow <TMessage, IEventMetadata>); } }
public AppraisersFlow(IEnumerable <Appraiser> appraisers) : base(DataflowOptions.Default) { if (appraisers is null) { throw new ArgumentNullException(nameof(appraisers)); } _inputConsumer = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData => { Console.WriteLine($"Broadcasts all crawlers data. {crawlersData.Count.ToString()}"); return(crawlersData); }, DataflowOptions.Default); _resultConsumer = new TransformBlock <IReadOnlyList <string>, IReadOnlyList <string> >( appraisedData => appraisedData ).ToDataflow(DataflowOptions.Default); var usedTypes = new Dictionary <Type, DataBroadcaster <IReadOnlyList <IData> > >(); foreach (var appraiser in appraisers) { if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster)) { broadcaster = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData => { Console.WriteLine($"Broadcasts specified data of type {appraiser.DataType.Name}. {crawlersData.Count.ToString()}"); return(crawlersData); }, DataflowOptions.Default); usedTypes.Add(appraiser.DataType, broadcaster); _inputConsumer.TransformAndLink( broadcaster, l => l, l => l.All(d => d.GetType().IsAssignableFrom(appraiser.DataType)) ); RegisterChild(broadcaster); } var appraiserFlow = DataflowUtils.FromDelegate(appraiser.Func, DataflowOptions.Default); broadcaster.LinkTo(appraiserFlow); appraiserFlow.LinkTo(_resultConsumer); _resultConsumer.RegisterDependency(appraiserFlow); RegisterChild(appraiserFlow); } RegisterChild(_inputConsumer); RegisterChild(_resultConsumer); }
public OutputtersFlow(IEnumerable <Action <RatingDataContainer> > outputters) : base(DataflowOptions.Default) { outputters.ThrowIfNull(nameof(outputters)); _resultList = new ConcurrentBag <RatingDataContainer>(); _inputConsumer = new DataBroadcaster <RatingDataContainer>(appraisersData => { _resultList.Add(appraisersData); return(appraisersData); }, DataflowOptions.Default); InitFlow(outputters); }
public void OpenDataflow(DataflowSave ds) { Dataflow d = ds.ToDataflow(); d.save = ds; if (dataflow != null && dataflow.save != ds) { CloseDataflow(); } SetReadonly(false); OpenDataflow(d); }
public AppraisersFlow(IEnumerable <Funcotype> appraisers) : base(DataflowOptions.Default) { appraisers.ThrowIfNull(nameof(appraisers)); _inputConsumer = new DataBroadcaster <BasicInfo>( crawlersData => crawlersData, DataflowOptions.Default ); _resultConsumer = new TransformBlock <RatingDataContainer, RatingDataContainer>( appraisedData => appraisedData ).ToDataflow(DataflowOptions.Default); InitFlow(appraisers); }
public CrawlersFlow(IEnumerable <Func <string, IAsyncEnumerable <BasicInfo> > > crawlers) : base(DataflowOptions.Default) { crawlers.ThrowIfNull(nameof(crawlers)); _inputBroadcaster = new DataBroadcaster <string>( filteredData => filteredData, DataflowOptions.Default ); _resultConsumer = new TransformBlock <BasicInfo, BasicInfo>( crawlersData => crawlersData ).ToDataflow(DataflowOptions.Default); InitFlow(crawlers); }
public ComplexIntFlow() : base(DataflowOptions.Default) { Dataflow <int, int> node2 = DataflowUtils.FromDelegate <int, int>(i => i); Dataflow <int, int> node3 = DataflowUtils.FromDelegate <int, int>(i => i * -1); Dataflow <int, int> node1 = DataflowUtils.FromDelegate <int, int>( i => { if (i % 2 == 0) { node2.Post(i); } else { node3.Post(i); } return(999); }); Dataflow <int> printer = DataflowUtils.FromDelegate <int>(Console.WriteLine); node1.Name = "node1"; node2.Name = "node2"; node3.Name = "node3"; printer.Name = "printer"; this.RegisterChild(node1); this.RegisterChild(node2); this.RegisterChild(node3); this.RegisterChild(printer, t => { if (t.Status == TaskStatus.RanToCompletion) { Console.WriteLine("Printer done!"); } }); node1.LinkTo(printer); node2.LinkTo(printer); node3.LinkTo(printer); //Completion propagation: node1 ---> node2 node2.RegisterDependency(node1); //Completion propagation: node1 + node2 ---> node3 node3.RegisterDependency(node1); node3.RegisterDependency(node2); this._headBlock = node1.InputBlock; }
public SlowFlow(DataflowOptions dataflowOptions) : base(dataflowOptions) { _splitter = new TransformManyBlock <string, char>(new Func <string, IEnumerable <char> >(this.SlowSplit), dataflowOptions.ToExecutionBlockOption()) .ToDataflow(dataflowOptions, "SlowSplitter"); _printer = new ActionBlock <char>(c => Console.WriteLine(c), dataflowOptions.ToExecutionBlockOption()) .ToDataflow(dataflowOptions, "Printer"); RegisterChild(_splitter); RegisterChild(_printer); _splitter.LinkTo(_printer); }
public CircularFlow(DataflowOptions dataflowOptions) : base(dataflowOptions) { //a no-op node to demonstrate the usage of preTask param in RegisterChildRing _buffer = new BufferBlock <int>().ToDataflow(name: "NoOpBuffer"); var heater = new TransformManyDataflow <int, int>(async i => { await Task.Delay(200); Console.WriteLine("Heated to {0}", i + 1); return(new [] { i + 1 }); }, dataflowOptions); heater.Name = "Heater"; var cooler = new TransformManyDataflow <int, int>(async i => { await Task.Delay(200); int cooled = i - 2; Console.WriteLine("Cooled to {0}", cooled); if (cooled < 0) //time to stop { return(Enumerable.Empty <int>()); } return(new [] { cooled }); }, dataflowOptions); cooler.Name = "Cooler"; var heartbeat = new HeartbeatNode <int>(dataflowOptions) { Name = "HeartBeat" }; _buffer.LinkTo(heater); //circular heater.LinkTo(cooler); cooler.LinkTo(heartbeat); heartbeat.LinkTo(heater); RegisterChildren(_buffer, heater, cooler, heartbeat); //ring registration RegisterChildRing(_buffer.Completion, heater, cooler, heartbeat); }
/// <summary> /// Constructs a DbDataJoiner instance /// </summary> /// <param name="joinOn">Property path from the root object down to the lookup key</param> /// <param name="dimTableTarget">Table information of the remote dimension table</param> /// <param name="option">Option to use for this dataflow</param> /// <param name="batchSize">The batch size for a batched remote look up</param> /// <param name="cacheSize">The local cache item count (part of the remote table)</param> public DbDataJoiner(Expression <Func <TIn, TLookupKey> > joinOn, TargetTable dimTableTarget, DataflowOptions option, int batchSize = 8 * 1024, int cacheSize = 1024 * 1024) : base(option) { m_dimTableTarget = dimTableTarget; m_batchSize = batchSize; m_batcher = new BatchBlock <TIn>(batchSize, option.ToGroupingBlockOption()).ToDataflow(option); m_batcher.Name = "Batcher"; m_lookupNode = new TransformManyDataflow <JoinBatch <TIn>, TIn>(this.JoinBatch, option); m_lookupNode.Name = "LookupNode"; m_typeAccessor = TypeAccessorManager <TIn> .GetAccessorForTable(dimTableTarget); m_keyComparer = typeof(TLookupKey) == typeof(byte[]) ? (IEqualityComparer <TLookupKey>)((object)new ByteArrayEqualityComparer()) : EqualityComparer <TLookupKey> .Default; m_rowCache = new RowCache <TLookupKey>(cacheSize, m_keyComparer); m_logger = Utils.GetNamespaceLogger(); m_joinOnMapping = m_typeAccessor.DbColumnMappings.First(m => m.Host.PropertyInfo == this.ExtractPropertyInfo(joinOn)); var transformer = new TransformBlock <TIn[], JoinBatch <TIn> >( array => new JoinBatch <TIn>(array, CacheLookupStrategy.RemoteLookup), option.ToExecutionBlockOption()).ToDataflow(option); transformer.Name = "ArrayToJoinBatchConverter"; m_batcher.LinkTo(transformer); transformer.LinkTo(m_lookupNode); RegisterChild(m_batcher); RegisterChild(transformer); RegisterChild(m_lookupNode); m_dimInserter = new DimTableInserter(this, dimTableTarget, joinOn, option) { Name = "DimInserter" }; var hb = new HeartbeatNode <JoinBatch <TIn> >(option); m_dimInserter.RegisterDependency(m_lookupNode); m_dimInserter.LinkTo(hb); hb.LinkTo(m_lookupNode); RegisterChild(m_dimInserter); RegisterChild(hb); RegisterChildRing(transformer.CompletionTask, m_lookupNode, m_dimInserter, hb); }
public CompletionDemo1() { broadCaster = new BroadcastBlock<int>( i => { return i; }).ToDataflow(); transformBlock1 = new TransformBlock<int, string>( i => { Console.WriteLine("1 input count: " + transformBlock1.InputCount); Thread.Sleep(50); return ("1_" + i); }); transformBlock2 = new TransformBlock<int, string>( i => { Console.WriteLine("2 input count: " + transformBlock2.InputCount); Thread.Sleep(20); return ("2_" + i); }); processor = new ActionBlock<string>( i => { Console.WriteLine(i); }).ToDataflow(); /** rather than TPL linking broadCastBlock.LinkTo(transformBlock1, new DataflowLinkOptions { PropagateCompletion = true }); broadCastBlock.LinkTo(transformBlock2, new DataflowLinkOptions { PropagateCompletion = true }); transformBlock1.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true }); transformBlock2.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true }); **/ //Use DataflowEx linking var transform1 = transformBlock1.ToDataflow(); var transform2 = transformBlock2.ToDataflow(); broadCaster.LinkTo(transform1); broadCaster.LinkTo(transform2); transform1.LinkTo(processor); transform2.LinkTo(processor); }
public RingMonitor(Dataflow host, IRingNode[] ringNodes) { if (ringNodes.Length == 0) { throw new ArgumentException("The child ring contains nothing"); } m_hb = ringNodes.OfType<IHeartbeatNode>().FirstOrDefault(); if (m_hb == null) { throw new ArgumentException("A ring must contain at least one IHeartbeatNode"); } var nonChild = ringNodes.FirstOrDefault(r => { return !host.IsMyChild(r); }); if (nonChild != null) { throw new ArgumentException("The child ring contains a non-child: " + nonChild.FullName); } this.m_host = host; this.m_ringNodes = ringNodes; m_lazyDisplayName = new Lazy<string>(this.GetDisplayName); }
public NoChildRegisteredException(Dataflow dataflow) : base("No child has been registered in " + dataflow.FullName) { }