public void TestUdfSerialization()
{
Func <string, int> f = (s) => 1;
Func <int, IEnumerable <dynamic>, IEnumerable <dynamic> > udfHelper = new UdfHelper <int, string>(f).Execute;
var udfCommand = SparkContext.BuildCommand(new CSharpWorkerFunc(udfHelper), SerializedMode.String,
SerializedMode.String);
using (var outputStream = new MemoryStream(500))
using (var inputStream = new MemoryStream(500))
{
SerDe.Write(inputStream, "1.0"); //version
SerDe.Write(inputStream, 0);
SerDe.Write(inputStream, 0);
SerDe.Write(inputStream, 0);
SerDe.Write(inputStream, 0L);
SerDe.Write(inputStream, ""); //includes directory
SerDe.Write(inputStream, 0); //number of included items
SerDe.Write(inputStream, 0); //number of broadcast variables
SerDe.Write(inputStream, 1); //flag for UDF
SerDe.Write(inputStream, 1); //count of udfs
SerDe.Write(inputStream, 1); //count of args
SerDe.Write(inputStream, 0); //index of args
SerDe.Write(inputStream, 1); //count of chained func
SerDe.Write(inputStream, udfCommand.Length);
SerDe.Write(inputStream, udfCommand);
SerDe.Write(inputStream, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(inputStream, (int)SpecialLengths.END_OF_STREAM);
inputStream.Flush();
inputStream.Position = 0;
Worker.InitializeLogger();
Worker.ProcessStream(inputStream, outputStream, 1);
outputStream.Position = 0;
foreach (var val in ReadWorker(outputStream))
{
//Section in output could be successfuly read from the stream
}
}
}
public bool Parse(int start, byte[] buffer, int size, ref int processed)
{
processed = 0;
if (size < 38)
{
return(false);
}
VolumeTag tag = new VolumeTag();
tag.Parse(start, buffer, size);
if (tag.Identifier != (short)VolumeDescriptorType.FileId)
{
return(false);
}
_fileCharacteristics = buffer[start + 18];
int idLen = buffer[start + 19];
this.Icb.Parse(start + 20, buffer);
int impLen = UdfHelper.Get16(start + 36, buffer);
if (size < 38 + idLen + impLen)
{
return(false);
}
processed = 38;
processed += impLen;
this.Identifier.Parse(start + processed, buffer, idLen);
processed += idLen;
for (; (processed & 3) != 0; processed++)
{
if (buffer[start + processed] != 0)
{
return(false);
}
}
return(processed <= size);
}
public void Parse(int start, byte[] buffer)
{
this.Length = UdfHelper.Get32(start, buffer);
this.Location.Parse(start + 4, buffer);
}
public void Parse(int start, byte[] buffer)
{
this.Position = UdfHelper.Get32(start, buffer);
this.PartitionReference = UdfHelper.Get16(start + 4, buffer);
}
