public void ChainTest()
{
var func1 = new CSharpWorkerFunc((id, iter) => new List <dynamic> {
1, 2, 3
});
var func2 = new CSharpWorkerFunc(Multiplier);
var func3 = CSharpWorkerFunc.Chain(func1, func2); //func1 will be executed first on input and result will be input to func2
var result = func3.Func(1, new List <dynamic>()).Cast <int>().ToArray();
Assert.AreEqual(10, result[0]);
Assert.AreEqual(20, result[1]);
Assert.AreEqual(30, result[2]);
}
private static IFormatter ProcessCommand(Stream inputStream, Stream outputStream, int splitIndex, DateTime bootTime)
{
int isSqlUdf = SerDe.ReadInt(inputStream);
logger.LogDebug("Is func Sql UDF = {0}", isSqlUdf);
IFormatter formatter = new BinaryFormatter();
if (isSqlUdf == 0)
{
logger.LogDebug("Processing non-UDF command");
int lengthOfCommandByteArray = SerDe.ReadInt(inputStream);
logger.LogDebug("Command length: " + lengthOfCommandByteArray);
if (lengthOfCommandByteArray > 0)
{
var commandProcessWatch = new Stopwatch();
commandProcessWatch.Start();
int stageId;
string deserializerMode;
string serializerMode;
CSharpWorkerFunc workerFunc;
ReadCommand(inputStream, formatter, out stageId, out deserializerMode, out serializerMode,
out workerFunc);
ExecuteCommand(inputStream, outputStream, splitIndex, bootTime, deserializerMode, workerFunc, serializerMode,
formatter, commandProcessWatch, stageId, isSqlUdf);
}
else
{
logger.LogWarn("lengthOfCommandByteArray = 0. Nothing to execute :-(");
}
}
else
{
logger.LogDebug("Processing UDF command");
var udfCount = SerDe.ReadInt(inputStream);
logger.LogDebug("Count of UDFs = {0}", udfCount);
if (udfCount == 1)
{
CSharpWorkerFunc func = null;
var argCount = SerDe.ReadInt(inputStream);
logger.LogDebug("Count of args = {0}", argCount);
var argOffsets = new List <int>();
for (int argIndex = 0; argIndex < argCount; argIndex++)
{
var offset = SerDe.ReadInt(inputStream);
logger.LogDebug("UDF argIndex = {0}, Offset = {1}", argIndex, offset);
argOffsets.Add(offset);
}
var chainedFuncCount = SerDe.ReadInt(inputStream);
logger.LogDebug("Count of chained func = {0}", chainedFuncCount);
var commandProcessWatch = new Stopwatch();
int stageId = -1;
string deserializerMode = null;
string serializerMode = null;
CSharpWorkerFunc workerFunc = null;
for (int funcIndex = 0; funcIndex < chainedFuncCount; funcIndex++)
{
int lengthOfCommandByteArray = SerDe.ReadInt(inputStream);
logger.LogDebug("UDF command length: " + lengthOfCommandByteArray)
;
if (lengthOfCommandByteArray > 0)
{
ReadCommand(inputStream, formatter, out stageId, out deserializerMode, out serializerMode,
out workerFunc);
if (func == null)
{
func = workerFunc;
}
else
{
func = CSharpWorkerFunc.Chain(func, workerFunc);
}
}
else
{
logger.LogWarn("UDF lengthOfCommandByteArray = 0. Nothing to execute :-(");
}
}
Debug.Assert(stageId != -1);
Debug.Assert(deserializerMode != null);
Debug.Assert(serializerMode != null);
Debug.Assert(func != null);
ExecuteCommand(inputStream, outputStream, splitIndex, bootTime, deserializerMode, func, serializerMode, formatter,
commandProcessWatch, stageId, isSqlUdf);
}
else
{
throw new NotSupportedException(); //TODO - add support for multiple UDFs
}
}
return(formatter);
}
private static UDFCommand ProcessUdfCommand(Stream inputStream, Stream outputStream, int splitIndex,
DateTime bootTime, IFormatter formatter, int isSqlUdf)
{
logger.LogDebug("Processing UDF command");
var udfCount = SerDe.ReadInt(inputStream);
logger.LogDebug("Count of UDFs = {0}", udfCount);
int stageId = -1;
string deserializerMode = null;
string serializerMode = null;
var commandProcessWatch = new Stopwatch();
List <WorkerFunc> workerFuncList = new List <WorkerFunc>();
for (int udfIter = 0; udfIter < udfCount; udfIter++)
{
CSharpWorkerFunc func = null;
var argCount = SerDe.ReadInt(inputStream);
logger.LogDebug("Count of args = {0}", argCount);
List <int> argOffsets = new List <int>();
for (int argIndex = 0; argIndex < argCount; argIndex++)
{
var offset = SerDe.ReadInt(inputStream);
logger.LogDebug("UDF argIndex = {0}, Offset = {1}", argIndex, offset);
argOffsets.Add(offset);
}
var chainedFuncCount = SerDe.ReadInt(inputStream);
logger.LogDebug("Count of chained func = {0}", chainedFuncCount);
for (int funcIndex = 0; funcIndex < chainedFuncCount; funcIndex++)
{
int lengthOfCommandByteArray = SerDe.ReadInt(inputStream);
logger.LogDebug("UDF command length: " + lengthOfCommandByteArray);
if (lengthOfCommandByteArray > 0)
{
CSharpWorkerFunc workerFunc;
ReadCommand(inputStream, formatter, out stageId, out deserializerMode, out serializerMode,
out workerFunc);
func = func == null ? workerFunc : CSharpWorkerFunc.Chain(func, workerFunc);
}
else
{
logger.LogWarn("UDF lengthOfCommandByteArray = 0. Nothing to execute :-(");
}
}
Debug.Assert(stageId != -1);
Debug.Assert(deserializerMode != null);
Debug.Assert(serializerMode != null);
Debug.Assert(func != null);
workerFuncList.Add(new WorkerFunc(func, argCount, argOffsets));
}
return(new UDFCommand(inputStream, outputStream, splitIndex, bootTime, deserializerMode,
serializerMode, formatter, commandProcessWatch, isSqlUdf, workerFuncList, stageId));
}
