public static object ApplyMerger <TKeyI2, TPayloadI2, TDatasetI2, TKeyO, TPayloadO, TDatasetO>(
object[] datasets, IMoveDescriptor moveDescriptor, string merger)
where TDatasetI2 : IDataset <TKeyI2, TPayloadI2>
where TDatasetO : IDataset <TKeyO, TPayloadO>
{
try
{
if (merger != null)
{
TDatasetI2[] transientDatasets = new TDatasetI2[datasets.Length];
for (int i = 0; i < transientDatasets.Length; i++)
{
transientDatasets[i] = (TDatasetI2)datasets[i];
}
Expression transformer = SerializationHelper.Deserialize(merger);
var compiledTransformer = Expression <Func <TDatasetI2[], IMoveDescriptor, TDatasetO> > .Lambda(transformer).Compile();
Delegate compiledTransformerConstructor = (Delegate)compiledTransformer.DynamicInvoke();
return(compiledTransformerConstructor.DynamicInvoke(transientDatasets, moveDescriptor));
}
}
catch (Exception e)
{
Console.WriteLine("Error: The CRA vertex failed to apply a mergere transform for an error of type " + e.GetType() + " : " + e.ToString());
}
return(null);
}
public IShardedDataset <TKeyO, TPayloadO, TDatasetO> Move <TKey2, TPayload2, TDataset2, TKeyO, TPayloadO, TDatasetO>(
Expression <Func <TDataset, IMoveDescriptor, TDataset2[]> > mapper,
Expression <Func <TDataset2[], IMoveDescriptor, TDatasetO> > reducer, IMoveDescriptor moveDescriptor)
where TDataset2 : IDataset <TKey2, TPayload2>
where TDatasetO : IDataset <TKeyO, TPayloadO>
{
return(new DeployableShardedDataset <TKey, TPayload, TDataset, TKey2, TPayload2, TDataset2, TKeyO, TPayloadO, TDatasetO>(
this, mapper, reducer, moveDescriptor));
}
internal DeployableShardedDataset(IShardedDataset <TKeyI1, TPayloadI1, TDataSetI1> input,
Expression <Func <TDataSetI1, TDataSetO> > transform)
{
_input1 = input;
_input2 = null;
_operationType = OperatorType.UnaryTransform;
var closureEliminator = new ClosureEliminator();
_unaryTransformer = closureEliminator.Visit(transform) as Expression <Func <TDataSetI1, TDataSetO> >;
_binaryTransformer = null;
_splitter = null;
_merger = null;
_moveDescriptor = null;
}
internal DeployableShardedDataset(
IDataProvider dataProvider,
IShardedDataset <TKeyI1, TPayloadI1, TDataSetI1> input,
Expression <Func <TDataSetI1, IMoveDescriptor, TDataSetI2[]> > splitter,
Expression <Func <TDataSetI2[], IMoveDescriptor, TDataSetO> > merger, IMoveDescriptor moveDescriptor)
: base(dataProvider)
{
_input1 = input;
_input2 = null;
_operationType = OperatorType.Move;
var closureEliminator = new ClosureEliminator();
_splitter = closureEliminator.Visit(splitter) as Expression <Func <TDataSetI1, IMoveDescriptor, TDataSetI2[]> >;
_merger = closureEliminator.Visit(merger) as Expression <Func <TDataSetI2[], IMoveDescriptor, TDataSetO> >;
_moveDescriptor = moveDescriptor;
_unaryTransformer = null;
_binaryTransformer = null;
_dataProvider = dataProvider;
}
internal DeployableShardedDataset(
IDataProvider dataProvider,
IShardedDataset <TKeyI1, TPayloadI1, TDataSetI1> input1,
IShardedDataset <TKeyI2, TPayloadI2, TDataSetI2> input2,
Expression <Func <TDataSetI1, TDataSetI2, TDataSetO> > transform)
: base(dataProvider)
{
_input1 = input1;
_input2 = input2;
_operationType = OperatorType.BinaryTransform;
var closureEliminator = new ClosureEliminator();
_binaryTransformer = closureEliminator.Visit(transform) as Expression <Func <TDataSetI1, TDataSetI2, TDataSetO> >;
_unaryTransformer = null;
_splitter = null;
_merger = null;
_moveDescriptor = null;
_dataProvider = dataProvider;
}
public static object ApplySplitter <TKeyI1, TPayloadI1, TDatasetI1, TKeyI2, TPayloadI2, TDatasetI2>(
object dataset, IMoveDescriptor moveDescriptor, string splitter)
where TDatasetI1 : IDataset <TKeyI1, TPayloadI1>
where TDatasetI2 : IDataset <TKeyI2, TPayloadI2>
{
try
{
if (splitter != null)
{
var transformer = (Expression <Func <TDatasetI1, IMoveDescriptor, TDatasetI2[]> >)SerializationHelper.Deserialize(splitter);
var compiledTransformer = transformer.Compile();
return(compiledTransformer((TDatasetI1)dataset, moveDescriptor));
}
}
catch (Exception e)
{
Console.WriteLine("Error: the CRA vertex failed to apply a splitter transform for an error of type " + e.GetType() + ": " + e.ToString());
}
return(null);
}
public Expression <Func <IDataset <TKey, TPayload>[], IDataset <TKey, TPayload> > > CreateMerger(IMoveDescriptor descriptor)
{
throw new NotImplementedException();
}
public Expression <Func <IDataset <Empty, Empty>[], IDataset <Empty, Empty> > > CreateMerger(IMoveDescriptor moveDescriptor)
{
return(null);
}
public Expression <Func <IDataset <Empty, Empty>, IDataset <Empty, Empty>[]> > CreateSplitter(IMoveDescriptor moveDescriptor)
{
return(null);
}
public static IntKeyedDataset <int, int> Merger(this IntKeyedDataset <int, int>[] source, IMoveDescriptor descriptor)
{
return(new IntKeyedDataset <int, int>(source[0]._shift + source[1]._shift));
}
public static IntKeyedDataset <int, int>[] Splitter(this IntKeyedDataset <int, int> source, IMoveDescriptor descriptor)
{
IntKeyedDataset <int, int>[] outputs = new IntKeyedDataset <int, int> [2];
outputs[0] = new IntKeyedDataset <int, int>(source._shift + 100);
outputs[1] = new IntKeyedDataset <int, int>(source._shift + 200);
return(outputs);
}
public ShuffleTask(IMoveDescriptor shuffleDescriptor) : base(OperatorType.Move)
{
ShuffleDescriptor = shuffleDescriptor;
}
