public void TestUdfSerialization() { Func <string, int> f = (s) => 1; Func <int, IEnumerable <dynamic>, IEnumerable <dynamic> > udfHelper = new UdfHelper <int, string>(f).Execute; var udfCommand = SparkContext.BuildCommand(new CSharpWorkerFunc(udfHelper), SerializedMode.String, SerializedMode.String); using (var outputStream = new MemoryStream(500)) using (var inputStream = new MemoryStream(500)) { SerDe.Write(inputStream, "1.0"); //version SerDe.Write(inputStream, 0); SerDe.Write(inputStream, 0); SerDe.Write(inputStream, 0); SerDe.Write(inputStream, 0L); SerDe.Write(inputStream, ""); //includes directory SerDe.Write(inputStream, 0); //number of included items SerDe.Write(inputStream, 0); //number of broadcast variables SerDe.Write(inputStream, 1); //flag for UDF SerDe.Write(inputStream, 1); //count of udfs SerDe.Write(inputStream, 1); //count of args SerDe.Write(inputStream, 0); //index of args SerDe.Write(inputStream, 1); //count of chained func SerDe.Write(inputStream, udfCommand.Length); SerDe.Write(inputStream, udfCommand); SerDe.Write(inputStream, (int)SpecialLengths.END_OF_DATA_SECTION); SerDe.Write(inputStream, (int)SpecialLengths.END_OF_STREAM); inputStream.Flush(); inputStream.Position = 0; Worker.InitializeLogger(); Worker.ProcessStream(inputStream, outputStream, 1); outputStream.Position = 0; foreach (var val in ReadWorker(outputStream)) { //Section in output could be successfuly read from the stream } } }
public bool Parse(int start, byte[] buffer, int size, ref int processed) { processed = 0; if (size < 38) { return(false); } VolumeTag tag = new VolumeTag(); tag.Parse(start, buffer, size); if (tag.Identifier != (short)VolumeDescriptorType.FileId) { return(false); } _fileCharacteristics = buffer[start + 18]; int idLen = buffer[start + 19]; this.Icb.Parse(start + 20, buffer); int impLen = UdfHelper.Get16(start + 36, buffer); if (size < 38 + idLen + impLen) { return(false); } processed = 38; processed += impLen; this.Identifier.Parse(start + processed, buffer, idLen); processed += idLen; for (; (processed & 3) != 0; processed++) { if (buffer[start + processed] != 0) { return(false); } } return(processed <= size); }
public void Parse(int start, byte[] buffer) { this.Length = UdfHelper.Get32(start, buffer); this.Location.Parse(start + 4, buffer); }
public void Parse(int start, byte[] buffer) { this.Position = UdfHelper.Get32(start, buffer); this.PartitionReference = UdfHelper.Get16(start + 4, buffer); }