Beispiel #1
0
        public static UnaryOperatorTypes FillUnaryTransformTypes(Type tKeyI1, Type tPayloadI1,
                                                                 Type tDatasetI1, Type tKeyO, Type tPayloadO, Type tDatasetO)
        {
            var unaryTransformTypes = new UnaryOperatorTypes();

            unaryTransformTypes.InputKeyType     = tKeyI1;
            unaryTransformTypes.InputPayloadType = tPayloadI1;
            unaryTransformTypes.InputDatasetType = tDatasetI1;

            unaryTransformTypes.OutputKeyType     = tKeyO;
            unaryTransformTypes.OutputPayloadType = tPayloadO;
            unaryTransformTypes.OutputDatasetType = tDatasetO;

            return(unaryTransformTypes);
        }
Beispiel #2
0
        private void ApplyTransformersOnInputs()
        {
            if (_vertex._task.Transforms != null)
            {
                MethodInfo method = null; MethodInfo generic = null; object[] arguments = null;

                for (int i = 0; i < _vertex._task.Transforms.Length; i++)
                {
                    object dataset1 = null; string dataset1Id = null;
                    object dataset2 = null; string dataset2Id = null;
                    TransformUtils.PrepareTransformInputs(_vertex._task.TransformsInputs[i], ref dataset1, ref dataset1Id,
                                                          ref dataset2, ref dataset2Id, _vertex._cachedDatasets[_shardId]);

                    string transformType   = _vertex._task.TransformsOperations[i];
                    object transformOutput = null;
                    if (transformType == OperatorType.UnaryTransform.ToString())
                    {
                        UnaryOperatorTypes unaryTransformTypes = new UnaryOperatorTypes();
                        unaryTransformTypes.FromString(_vertex._task.TransformsTypes[i]);
                        if (dataset1Id == "$" && dataset1 == null)
                        {
                            throw new InvalidOperationException();
                        }

                        method  = typeof(TransformUtils).GetMethod("ApplyUnaryTransformer");
                        generic = method.MakeGenericMethod(
                            new Type[] { unaryTransformTypes.InputKeyType, unaryTransformTypes.InputPayloadType,
                                         unaryTransformTypes.InputDatasetType, unaryTransformTypes.OutputKeyType,
                                         unaryTransformTypes.OutputPayloadType, unaryTransformTypes.OutputDatasetType });
                        arguments = new Object[] { dataset1, _vertex._task.Transforms[i] };

                        _vertex._outputKeyType     = unaryTransformTypes.OutputKeyType;
                        _vertex._outputPayloadType = unaryTransformTypes.OutputPayloadType;
                        _vertex._outputDatasetType = unaryTransformTypes.OutputDatasetType;
                    }
                    else if (transformType == OperatorType.BinaryTransform.ToString())
                    {
                        BinaryOperatorTypes binaryTransformTypes = new BinaryOperatorTypes();
                        binaryTransformTypes.FromString(_vertex._task.TransformsTypes[i]);
                        if (dataset1Id == "$" && dataset1 == null)
                        {
                            throw new InvalidOperationException();
                        }
                        if (dataset2Id == "$" && dataset2 == null)
                        {
                            dataset2Id = _vertex._task.TransformsInputs[i].InputId2;
                            _vertex._binaryOperatorTypes[dataset2Id] = binaryTransformTypes;

                            _vertex._startCreatingSecondaryDatasets[dataset2Id].Signal();
                            _vertex._finishCreatingSecondaryDatasets[dataset2Id].Wait();

                            dataset2 = _vertex._cachedDatasets[_shardId][dataset2Id];
                        }

                        method  = typeof(TransformUtils).GetMethod("ApplyBinaryTransformer");
                        generic = method.MakeGenericMethod(
                            new Type[] { binaryTransformTypes.InputKeyType, binaryTransformTypes.InputPayloadType,
                                         binaryTransformTypes.InputDatasetType, binaryTransformTypes.SecondaryKeyType,
                                         binaryTransformTypes.SecondaryPayloadType, binaryTransformTypes.SecondaryDatasetType,
                                         binaryTransformTypes.OutputKeyType, binaryTransformTypes.OutputPayloadType,
                                         binaryTransformTypes.OutputDatasetType });
                        arguments = new Object[] { dataset1, dataset2, _vertex._task.Transforms[i] };

                        _vertex._outputKeyType     = binaryTransformTypes.OutputKeyType;
                        _vertex._outputPayloadType = binaryTransformTypes.OutputPayloadType;
                        _vertex._outputDatasetType = binaryTransformTypes.OutputDatasetType;
                    }
                    else if (transformType == OperatorType.MoveSplit.ToString())
                    {
                        BinaryOperatorTypes splitTypes = new BinaryOperatorTypes();
                        splitTypes.FromString(_vertex._task.TransformsTypes[i]);
                        if (dataset1Id == "$" && dataset1 == null)
                        {
                            throw new InvalidOperationException();
                        }

                        method  = typeof(MoveUtils).GetMethod("ApplySplitter");
                        generic = method.MakeGenericMethod(
                            new Type[] { splitTypes.InputKeyType, splitTypes.InputPayloadType,
                                         splitTypes.InputDatasetType, splitTypes.SecondaryKeyType,
                                         splitTypes.SecondaryPayloadType, splitTypes.SecondaryDatasetType });
                        arguments = new Object[] { dataset1, _vertex._task.SecondaryShuffleDescriptor, _vertex._task.Transforms[i] };

                        _vertex._outputKeyType     = splitTypes.SecondaryKeyType;
                        _vertex._outputPayloadType = splitTypes.SecondaryPayloadType;
                        _vertex._outputDatasetType = splitTypes.SecondaryDatasetType;
                    }
                    else
                    {
                        throw new InvalidOperationException("Error: Unsupported transformation type");
                    }

                    if (method != null && generic != null && arguments != null)
                    {
                        transformOutput = generic.Invoke(this, arguments);
                    }
                    if (transformOutput != null)
                    {
                        if (!_vertex._cachedDatasets[_shardId].ContainsKey(dataset1Id))
                        {
                            _vertex._cachedDatasets[_shardId].Add(dataset1Id, transformOutput);
                        }
                        else
                        {
                            _vertex._cachedDatasets[_shardId][dataset1Id] = transformOutput;
                        }
                    }

                    _vertex._outputId = dataset1Id;
                }
            }
        }
Beispiel #3
0
        private async void StartMergeAndTransform()
        {
            var shuffleTask = (ShuffleTask)_task;

            BinaryOperatorTypes mergeTypes = new BinaryOperatorTypes();

            mergeTypes.FromString(shuffleTask.ShuffleTransformsTypes[1]);
            MethodInfo method  = typeof(MoveUtils).GetMethod("ApplyMerger");
            MethodInfo generic = method.MakeGenericMethod(
                new Type[] { mergeTypes.SecondaryKeyType, mergeTypes.SecondaryPayloadType,
                             mergeTypes.SecondaryDatasetType, mergeTypes.OutputKeyType,
                             mergeTypes.OutputPayloadType, mergeTypes.OutputDatasetType });

            object[] inputSplitDatasets = new object[_inputs.Length];
            for (int i = 0; i < _inputs.Length; i++)
            {
                int splitIndex = Convert.ToInt32(_inputEndpointOperatorIndex[i].Substring(_inputEndpointOperatorIndex[i].Length - 1));
                inputSplitDatasets[splitIndex] = CreateDatasetFromInput(_inputEndpointOperatorIndex[i], mergeTypes.SecondaryKeyType, mergeTypes.SecondaryPayloadType,
                                                                        mergeTypes.SecondaryDatasetType);
            }
            object[] arguments = new Object[] { inputSplitDatasets, shuffleTask.ShuffleDescriptor, shuffleTask.ShuffleTransforms[1] };
            _cachedDatasets[shuffleTask.OutputId] = generic.Invoke(this, arguments);

            _outputKeyType     = mergeTypes.OutputKeyType;
            _outputPayloadType = mergeTypes.OutputPayloadType;
            _outputDatasetType = mergeTypes.OutputDatasetType;
            _outputId          = shuffleTask.OutputId;

            if (_task.Transforms != null)
            {
                for (int i = 0; i < _task.Transforms.Length; i++)
                {
                    object dataset1 = null; string dataset1Id = null;
                    object dataset2 = null; string dataset2Id = null;
                    TransformUtils.PrepareTransformInputs(_task.TransformsInputs[i], ref dataset1, ref dataset1Id,
                                                          ref dataset2, ref dataset2Id, _cachedDatasets);

                    string transformType   = _task.TransformsOperations[i];
                    object transformOutput = null;
                    if (transformType == OperatorType.UnaryTransform.ToString())
                    {
                        UnaryOperatorTypes unaryTransformTypes = new UnaryOperatorTypes();
                        unaryTransformTypes.FromString(_task.TransformsTypes[i]);
                        if (dataset1Id == "$" && dataset1 == null)
                        {
                            throw new InvalidOperationException();
                        }

                        method  = typeof(TransformUtils).GetMethod("ApplyUnaryTransformer");
                        generic = method.MakeGenericMethod(
                            new Type[] { unaryTransformTypes.InputKeyType, unaryTransformTypes.InputPayloadType,
                                         unaryTransformTypes.InputDatasetType, unaryTransformTypes.OutputKeyType,
                                         unaryTransformTypes.OutputPayloadType, unaryTransformTypes.OutputDatasetType });
                        arguments = new Object[] { dataset1, _task.Transforms[i] };

                        _outputKeyType     = unaryTransformTypes.OutputKeyType;
                        _outputPayloadType = unaryTransformTypes.OutputPayloadType;
                        _outputDatasetType = unaryTransformTypes.OutputDatasetType;
                    }
                    else if (transformType == OperatorType.BinaryTransform.ToString())
                    {
                        BinaryOperatorTypes binaryTransformTypes = new BinaryOperatorTypes();
                        binaryTransformTypes.FromString(_task.TransformsTypes[i]);
                        if (dataset1Id == "$" && dataset1 == null)
                        {
                            throw new InvalidOperationException();
                        }
                        if (dataset2Id == "$" && dataset2 == null)
                        {
                            dataset2Id = _task.TransformsInputs[i].InputId2;
                            dataset2   = CreateDatasetFromSecondaryInput(dataset2Id, binaryTransformTypes.SecondaryKeyType,
                                                                         binaryTransformTypes.SecondaryPayloadType, binaryTransformTypes.SecondaryDatasetType);
                            if (!_cachedDatasets.ContainsKey(dataset2Id))
                            {
                                _cachedDatasets.Add(dataset2Id, dataset2);
                            }
                            else
                            {
                                _cachedDatasets[dataset2Id] = dataset2;
                            }
                        }

                        method  = typeof(TransformUtils).GetMethod("ApplyBinaryTransformer");
                        generic = method.MakeGenericMethod(
                            new Type[] { binaryTransformTypes.InputKeyType, binaryTransformTypes.InputPayloadType,
                                         binaryTransformTypes.InputDatasetType, binaryTransformTypes.SecondaryKeyType,
                                         binaryTransformTypes.SecondaryPayloadType, binaryTransformTypes.SecondaryDatasetType,
                                         binaryTransformTypes.OutputKeyType, binaryTransformTypes.OutputPayloadType,
                                         binaryTransformTypes.OutputDatasetType });
                        arguments = new Object[] { dataset1, dataset2, _task.Transforms[i] };

                        _outputKeyType     = binaryTransformTypes.OutputKeyType;
                        _outputPayloadType = binaryTransformTypes.OutputPayloadType;
                        _outputDatasetType = binaryTransformTypes.OutputDatasetType;
                    }
                    else if (transformType == OperatorType.MoveSplit.ToString())
                    {
                        BinaryOperatorTypes splitTypes = new BinaryOperatorTypes();
                        splitTypes.FromString(_task.TransformsTypes[i]);
                        if (dataset1Id == "$" && dataset1 == null)
                        {
                            throw new InvalidOperationException();
                        }

                        method  = typeof(MoveUtils).GetMethod("ApplySplitter");
                        generic = method.MakeGenericMethod(
                            new Type[] { splitTypes.InputKeyType, splitTypes.InputPayloadType,
                                         splitTypes.InputDatasetType, splitTypes.SecondaryKeyType,
                                         splitTypes.SecondaryPayloadType, splitTypes.SecondaryDatasetType });
                        arguments = new Object[] { dataset1, _task.SecondaryShuffleDescriptor, _task.Transforms[i] };

                        _outputKeyType     = splitTypes.SecondaryKeyType;
                        _outputPayloadType = splitTypes.SecondaryPayloadType;
                        _outputDatasetType = splitTypes.SecondaryDatasetType;
                    }
                    else
                    {
                        throw new InvalidOperationException("Error: Unsupported transformation type");
                    }

                    if (method != null && generic != null && arguments != null)
                    {
                        transformOutput = generic.Invoke(this, arguments);
                    }
                    if (transformOutput != null)
                    {
                        if (!_cachedDatasets.ContainsKey(dataset1Id))
                        {
                            _cachedDatasets.Add(dataset1Id, transformOutput);
                        }
                        else
                        {
                            _cachedDatasets[dataset1Id] = transformOutput;
                        }
                    }

                    _outputId = dataset1Id;
                }
            }

            await Task.Run(() => ApplyProducer());

            _isProduceAllowedToApply = false;
            _firstProduceTrigger.Set();
        }
Beispiel #4
0
        public void CreateAndTransformDataset(int shardId)
        {
            var produceTask = (ProduceTask)_task;

            MethodInfo method  = typeof(ShardedProducerOperator).GetMethod("CreateDatasetFromExpression");
            MethodInfo generic = method.MakeGenericMethod(
                new Type[] { produceTask.OperationTypes.OutputKeyType,
                             produceTask.OperationTypes.OutputPayloadType,
                             produceTask.OperationTypes.OutputDatasetType });

            object[] arguments = new Object[] { shardId, produceTask.DataProducer };
            _cachedDatasets[shardId][produceTask.OutputId] = generic.Invoke(this, arguments);

            _outputKeyType     = produceTask.OperationTypes.OutputKeyType;
            _outputPayloadType = produceTask.OperationTypes.OutputPayloadType;
            _outputDatasetType = produceTask.OperationTypes.OutputDatasetType;
            _outputId          = produceTask.OutputId;

            if (_task.Transforms != null)
            {
                for (int i = 0; i < _task.Transforms.Length; i++)
                {
                    object dataset1 = null; string dataset1Id = null;
                    object dataset2 = null; string dataset2Id = null;
                    TransformUtils.PrepareTransformInputs(_task.TransformsInputs[i], ref dataset1, ref dataset1Id,
                                                          ref dataset2, ref dataset2Id, _cachedDatasets[shardId]);

                    string transformType   = _task.TransformsOperations[i];
                    object transformOutput = null;
                    if (transformType == OperatorType.UnaryTransform.ToString())
                    {
                        UnaryOperatorTypes unaryTransformTypes = new UnaryOperatorTypes();
                        unaryTransformTypes.FromString(_task.TransformsTypes[i]);
                        if (dataset1Id == "$" && dataset1 == null)
                        {
                            throw new InvalidOperationException();
                        }

                        method  = typeof(TransformUtils).GetMethod("ApplyUnaryTransformer");
                        generic = method.MakeGenericMethod(
                            new Type[] { unaryTransformTypes.InputKeyType, unaryTransformTypes.InputPayloadType,
                                         unaryTransformTypes.InputDatasetType, unaryTransformTypes.OutputKeyType,
                                         unaryTransformTypes.OutputPayloadType, unaryTransformTypes.OutputDatasetType });
                        arguments = new Object[] { dataset1, _task.Transforms[i] };

                        _outputKeyType     = unaryTransformTypes.OutputKeyType;
                        _outputPayloadType = unaryTransformTypes.OutputPayloadType;
                        _outputDatasetType = unaryTransformTypes.OutputDatasetType;
                    }
                    else if (transformType == OperatorType.BinaryTransform.ToString())
                    {
                        BinaryOperatorTypes binaryTransformTypes = new BinaryOperatorTypes();
                        binaryTransformTypes.FromString(_task.TransformsTypes[i]);
                        if (dataset1Id == "$" && dataset1 == null)
                        {
                            throw new InvalidOperationException();
                        }
                        if (dataset2Id == "$" && dataset2 == null)
                        {
                            dataset2Id = _task.TransformsInputs[i].InputId2;
                            _binaryOperatorTypes[dataset2Id] = binaryTransformTypes;

                            _startCreatingSecondaryDatasets[dataset2Id].Signal();
                            _finishCreatingSecondaryDatasets[dataset2Id].Wait();

                            dataset2 = _cachedDatasets[shardId][dataset2Id];
                        }

                        method  = typeof(TransformUtils).GetMethod("ApplyBinaryTransformer");
                        generic = method.MakeGenericMethod(
                            new Type[] { binaryTransformTypes.InputKeyType, binaryTransformTypes.InputPayloadType,
                                         binaryTransformTypes.InputDatasetType, binaryTransformTypes.SecondaryKeyType,
                                         binaryTransformTypes.SecondaryPayloadType, binaryTransformTypes.SecondaryDatasetType,
                                         binaryTransformTypes.OutputKeyType, binaryTransformTypes.OutputPayloadType,
                                         binaryTransformTypes.OutputDatasetType });
                        arguments = new Object[] { dataset1, dataset2, _task.Transforms[i] };

                        _outputKeyType     = binaryTransformTypes.OutputKeyType;
                        _outputPayloadType = binaryTransformTypes.OutputPayloadType;
                        _outputDatasetType = binaryTransformTypes.OutputDatasetType;
                    }
                    else if (transformType == OperatorType.MoveSplit.ToString())
                    {
                        BinaryOperatorTypes splitTypes = new BinaryOperatorTypes();
                        splitTypes.FromString(_task.TransformsTypes[i]);
                        if (dataset1Id == "$" && dataset1 == null)
                        {
                            throw new InvalidOperationException();
                        }

                        method  = typeof(MoveUtils).GetMethod("ApplySplitter");
                        generic = method.MakeGenericMethod(
                            new Type[] { splitTypes.InputKeyType, splitTypes.InputPayloadType,
                                         splitTypes.InputDatasetType, splitTypes.SecondaryKeyType,
                                         splitTypes.SecondaryPayloadType, splitTypes.SecondaryDatasetType });
                        arguments = new Object[] { dataset1, _task.SecondaryShuffleDescriptor, _task.Transforms[i] };

                        _outputKeyType     = splitTypes.SecondaryKeyType;
                        _outputPayloadType = splitTypes.SecondaryPayloadType;
                        _outputDatasetType = splitTypes.SecondaryDatasetType;
                    }
                    else
                    {
                        throw new InvalidOperationException("Error: Unsupported transformation type");
                    }

                    transformOutput = generic.Invoke(this, arguments);
                    if (transformOutput != null)
                    {
                        if (!_cachedDatasets[shardId].ContainsKey(dataset1Id))
                        {
                            _cachedDatasets[shardId].Add(dataset1Id, transformOutput);
                        }
                        else
                        {
                            _cachedDatasets[shardId][dataset1Id] = transformOutput;
                        }
                    }

                    _outputId = dataset1Id;
                }
            }
        }