Esempio n. 1
0
        /// <summary>
        /// GeneratePipeline_DataFlowSource_to_DataFlowDestination generates a TPL-DataFlowPipeline between two vertices of a graph.
        /// v_source.UserDefinedObjects[0] has to be Type of IDataFlowSource
        /// v_dest.UserDefinedObjects[0] has to be Type of IDataFlowDestination
        /// </summary>
        /// <param name="v_source"></param>
        /// <param name="v_dest"></param>
        /// <param name="ToCompleteCollection"></param>
        /// <param name="WatingForCompletitionCollection"></param>
        private void GeneratePipeline_DataFlowSource_to_DataFlowDestination(Vertex v_source, Vertex v_dest, ref List <object> ToCompleteCollection, ref List <object> WatingForCompletitionCollection, ref Dictionary <IDataFlowSource <DS>, object> DataFlowReaderCollection)
        {
            IDataFlowSource <DS>      t_b_source = (IDataFlowSource <DS>)v_source.UserDefinedObjects[0];
            IDataFlowDestination <DS> dest       = (IDataFlowDestination <DS>)v_dest.UserDefinedObjects[0];

            TransformBlock <DS, DS> t_b_dummy = new TransformBlock <DS, DS>(DS => { return(DS); }
                                                                            , new ExecutionDataflowBlockOptions {
                MaxDegreeOfParallelism = this.MaxDegreeOfParallelism
            });

            ToCompleteCollection.Add(t_b_dummy);
            v_dest.UserDefinedObjects.Add(t_b_dummy);
            DataFlowReaderCollection.Add(t_b_source, t_b_dummy);


            var bacthBlock = new BatchBlock <DS>(BatchSize);
            var DataFlowDestinationBlock = new ActionBlock <DS[]>(outp => dest.WriteBatch(outp));

            t_b_dummy.LinkTo(bacthBlock, linkOptions);
            bacthBlock.LinkTo(DataFlowDestinationBlock, linkOptions);

            t_b_dummy.Completion.ContinueWith(t => { bacthBlock.Complete(); });
            bacthBlock.Completion.ContinueWith(t => { DataFlowDestinationBlock.Complete(); });

            WatingForCompletitionCollection.Add(DataFlowDestinationBlock);
        }
Esempio n. 2
0
        /// <summary>
        /// GeneratePipeline_DataFlowSource_to_Transformation generates a TPL-DataFlowPipeline between two vertices of a graph.
        /// v_source.UserDefinedObjects[0] has to be Type of IDataFlowSource
        /// v_dest.UserDefinedObjects[0] has to be Type of IDataFlowTransformation
        /// </summary>
        /// <param name="v_source"></param>
        /// <param name="v_dest"></param>
        /// <param name="ToCompleteCollection"></param>
        /// <param name="WatingForCompletitionCollection"></param>
        private void GeneratePipeline_DataFlowSource_to_Transformation(Vertex v_source, Vertex v_dest, ref List <object> ToCompleteCollection, ref List <object> WatingForCompletitionCollection, ref Dictionary <IDataFlowSource <DS>, object> DataFlowReaderCollection)
        {
            var t_b_source = (IDataFlowSource <DS>)v_source.UserDefinedObjects[0];
            var t_b_dest   = (IPropagatorBlock <DS, DS>)null;

            // for case that object in vertex is missing
            if (v_dest.UserDefinedObjects == null || v_dest.UserDefinedObjects[0] == null)
            {
                throw new Exception(string.Format("Vertex needs any object. For example DBDestination."));
            }
            // for case that v_dest is type of RowTransformFunction
            else if (v_dest.UserDefinedObjects[0].GetType() == typeof(RowTransformation <DS>))
            {
                RowTransformation <DS> t = (RowTransformation <DS>)v_dest.UserDefinedObjects[0];
                t_b_dest = new TransformBlock <DS, DS>(t.RowTransformFunction
                                                       , new ExecutionDataflowBlockOptions {
                    MaxDegreeOfParallelism = this.MaxDegreeOfParallelism
                });
            }
            // for case that v_dest is type of RowTransformationMany
            else if (v_dest.UserDefinedObjects[0].GetType() == typeof(RowTransformationMany <DS>))
            {
                RowTransformationMany <DS> t_many = (RowTransformationMany <DS>)v_dest.UserDefinedObjects[0];
                t_b_dest = new TransformManyBlock <DS, DS>(t_many.RowTransformManyFunction
                                                           , new ExecutionDataflowBlockOptions {
                    MaxDegreeOfParallelism = this.MaxDegreeOfParallelism
                });
            }
            // for case that v_dest is type of RowTransformationMany
            else if (v_dest.UserDefinedObjects[0].GetType() == typeof(BroadCast <DS>))
            {
                BroadCast <DS> t_broadcast = (BroadCast <DS>)v_dest.UserDefinedObjects[0];
                t_b_dest = new BroadcastBlock <DS>(t_broadcast.TransformFunction
                                                   , new ExecutionDataflowBlockOptions {
                    MaxDegreeOfParallelism = this.MaxDegreeOfParallelism
                });
            }
            // for case that type is not implemented
            else
            {
                string CurrentMethodName = new System.Diagnostics.StackFrame(0, true).GetMethod().Name;
                throw new Exception(string.Format("Not implemented Type {0} in {1}.", v_dest.UserDefinedObjects[0].GetType(), CurrentMethodName));
            }

            ToCompleteCollection.Add(t_b_dest);
            v_dest.UserDefinedObjects.Add(t_b_dest);
            DataFlowReaderCollection.Add(t_b_source, t_b_dest);
        }
Esempio n. 3
0
        /// <summary>
        /// GeneratePipeline_Transformation_to_DataFlowDestination generates a TPL-DataFlowPipeline between two vertices of a graph.
        /// v_source.UserDefinedObjects[0] has to be Type of IDataFlowTransformation
        /// v_dest.UserDefinedObjects[0] has to be Type of IDataFlowDestination
        /// </summary>
        /// <param name="v_source"></param>
        /// <param name="v_dest"></param>
        /// <param name="ToCompleteCollection"></param>
        /// <param name="WatingForCompletitionCollection"></param>
        private void GeneratePipeline_Transformation_to_DataFlowDestination(Vertex v_source, Vertex v_dest, ref List <object> ToCompleteCollection, ref List <object> WatingForCompletitionCollection)
        {
            var t_b_source = (IPropagatorBlock <DS, DS>)v_source.UserDefinedObjects[1];

            IDataFlowDestination <DS> dest = (IDataFlowDestination <DS>)v_dest.UserDefinedObjects[0];

            var bacthBlock = new BatchBlock <DS>(BatchSize);
            var DataFlowDestinationBlock = new ActionBlock <DS[]>(outp => dest.WriteBatch(outp));


            if (v_source.UserDefinedObjects[0].GetType() == typeof(RowTransformation <DS>))
            {
                t_b_source = (TransformBlock <DS, DS>)v_source.UserDefinedObjects[1];
            }
            else if (v_source.UserDefinedObjects[0].GetType() == typeof(RowTransformationMany <DS>))
            {
                t_b_source = (TransformManyBlock <DS, DS>)v_source.UserDefinedObjects[1];
            }
            else if (v_source.UserDefinedObjects[0].GetType() == typeof(BroadCast <DS>))
            {
                t_b_source = (BroadcastBlock <DS>)v_source.UserDefinedObjects[1];
            }
            else
            {
                string CurrentMethodName = new System.Diagnostics.StackFrame(0, true).GetMethod().Name;
                throw new Exception(string.Format("Not implemented Type {0} in {1}.", v_source.UserDefinedObjects[1].GetType(), CurrentMethodName));
            }

            t_b_source.LinkTo(bacthBlock, linkOptions);
            bacthBlock.LinkTo(DataFlowDestinationBlock, linkOptions);

            t_b_source.Completion.ContinueWith(t => { bacthBlock.Complete(); });
            bacthBlock.Completion.ContinueWith(t => { DataFlowDestinationBlock.Complete(); });

            WatingForCompletitionCollection.Add(DataFlowDestinationBlock);
        }
Esempio n. 4
0
        /// <summary>
        /// GeneratePipeline_Transformation_to_Transformation generates a TPL-DataFlowPipeline between two vertices of a graph.
        /// v_source.UserDefinedObjects[0] and v_dest.UserDefinedObjects[0] has to be Type of IDataFlowTransformation - so its a pipeline between two transformations.
        /// </summary>
        /// <param name="v_source"></param>
        /// <param name="v_dest"></param>
        /// <param name="ToCompleteCollection"></param>
        /// <param name="WatingForCompletitionCollection"></param>
        private void GeneratePipeline_Transformation_to_Transformation(Vertex v_source, Vertex v_dest, ref List <object> ToCompleteCollection, ref List <object> WatingForCompletitionCollection)
        {
            var t_b_source = (IPropagatorBlock <DS, DS>)v_source.UserDefinedObjects[1];
            var t_b_dest   = (IPropagatorBlock <DS, DS>)null;


            if (v_dest.UserDefinedObjects[0].GetType() == typeof(RowTransformation <DS>))
            {
                RowTransformation <DS> tr = (RowTransformation <DS>)v_dest.UserDefinedObjects[0];
                t_b_dest = new TransformBlock <DS, DS>(tr.RowTransformFunction
                                                       , new ExecutionDataflowBlockOptions {
                    MaxDegreeOfParallelism = this.MaxDegreeOfParallelism
                });
            }
            else if (v_dest.UserDefinedObjects[0].GetType() == typeof(RowTransformationMany <DS>))
            {
                RowTransformationMany <DS> t_many = (RowTransformationMany <DS>)v_dest.UserDefinedObjects[0];
                t_b_dest = new TransformManyBlock <DS, DS>(t_many.RowTransformManyFunction
                                                           , new ExecutionDataflowBlockOptions {
                    MaxDegreeOfParallelism = this.MaxDegreeOfParallelism
                });
            }
            else if (v_dest.UserDefinedObjects[0].GetType() == typeof(BroadCast <DS>))
            {
                BroadCast <DS> t_broadcast = (BroadCast <DS>)v_dest.UserDefinedObjects[0];
                t_b_dest = new BroadcastBlock <DS>(t_broadcast.TransformFunction
                                                   , new ExecutionDataflowBlockOptions {
                    MaxDegreeOfParallelism = this.MaxDegreeOfParallelism
                });
            }
            else
            {
                string CurrentMethodName = new System.Diagnostics.StackFrame(0, true).GetMethod().Name;
                throw new Exception(string.Format("Not implemented Type {0} in {1}.", v_dest.UserDefinedObjects[0].GetType(), CurrentMethodName));
            }
            //ToCompleteCollection.Add(t_b_dest); // fuehrte zu Fehlern bei dotnet execute auf linux-system
            v_dest.UserDefinedObjects.Add(t_b_dest);

            t_b_source.LinkTo(t_b_dest, linkOptions);
            t_b_source.Completion.ContinueWith(t => { t_b_dest.Complete(); });

            WatingForCompletitionCollection.Add(t_b_dest);
        }