public void TestClosedStreamWithMemoryStream()
{
var inputStream = new MemoryStream();
// Version is not used in this scenario.
var processor = new PayloadProcessor(null);
// Nothing is written to the stream.
Assert.Null(processor.Process(inputStream));
inputStream.Dispose();
// The stream is closed. Payload with null is expected.
Assert.Null(processor.Process(inputStream));
}
public void TestClosedStreamWithSocket()
{
var commandPayload = new CommandPayload()
{
EvalType = UdfUtils.PythonEvalType.SQL_BATCHED_UDF,
Commands = new Command[] { }
};
PayloadWriter payloadWriter = new PayloadWriterFactory().Create();
Payload payload = TestData.GetDefaultPayload();
var serverListener = new DefaultSocketWrapper();
serverListener.Listen();
var port = (serverListener.LocalEndPoint as IPEndPoint).Port;
var clientSocket = new DefaultSocketWrapper();
clientSocket.Connect(IPAddress.Loopback, port, null);
using (ISocketWrapper serverSocket = serverListener.Accept())
{
Stream outStream = serverSocket.OutputStream;
payloadWriter.Write(outStream, payload, commandPayload);
outStream.Flush();
}
// At this point server socket is closed.
Stream inStream = clientSocket.InputStream;
// Consume bytes already written to the socket.
var payloadProcessor = new PayloadProcessor(payloadWriter.Version);
Payload actualPayload = payloadProcessor.Process(inStream);
Assert.Equal(payload.SplitIndex, actualPayload.SplitIndex);
Assert.Equal(payload.Version, actualPayload.Version);
Assert.Equal(payload.TaskContext, actualPayload.TaskContext);
Assert.Equal(payload.SparkFilesDir, actualPayload.SparkFilesDir);
Assert.Equal(payload.IncludeItems, actualPayload.IncludeItems);
Assert.Equal(payload.BroadcastVariables.Count, actualPayload.BroadcastVariables.Count);
ValidateCommandPayload(commandPayload, actualPayload.Command);
// Another read will detect that the socket is closed.
Assert.Null(payloadProcessor.Process(inStream));
}
public void TestPayloadProcessor(string version)
{
CommandPayload commandPayload = TestData.GetDefaultCommandPayload();
PayloadWriter payloadWriter = new PayloadWriterFactory().Create(new Version(version));
Payload payload = TestData.GetDefaultPayload();
Payload actualPayload = null;
using (var outStream = new MemoryStream())
{
payloadWriter.Write(outStream, payload, commandPayload);
using (var inputStream = new MemoryStream(outStream.ToArray()))
{
actualPayload =
new PayloadProcessor(payloadWriter.Version).Process(inputStream);
}
}
// Validate the read payload.
Assert.Equal(payload.SplitIndex, actualPayload.SplitIndex);
Assert.Equal(payload.Version, actualPayload.Version);
Assert.Equal(payload.TaskContext, actualPayload.TaskContext);
Assert.Equal(payload.SparkFilesDir, actualPayload.SparkFilesDir);
Assert.Equal(payload.IncludeItems, actualPayload.IncludeItems);
Assert.Equal(payload.BroadcastVariables.Count, actualPayload.BroadcastVariables.Count);
ValidateCommandPayload(commandPayload, actualPayload.Command);
// Validate the UDFs.
var actualCommand1 = (SqlCommand)actualPayload.Command.Commands[0];
var result1 = ((PicklingWorkerFunction)actualCommand1.WorkerFunction).Func(
0,
new object[] { "hello", 10, 20 },
actualCommand1.ArgOffsets);
Assert.Equal("udf2 udf1 hello", result1);
var actualCommand2 = (SqlCommand)actualPayload.Command.Commands[1];
var result2 = ((PicklingWorkerFunction)actualCommand2.WorkerFunction).Func(
0,
new object[] { "hello", 10, 20 },
actualCommand2.ArgOffsets);
Assert.Equal(30, result2);
}
private Payload ProcessStream(
Stream inputStream,
Stream outputStream,
Version version,
out bool readComplete)
{
readComplete = false;
try
{
DateTime bootTime = DateTime.UtcNow;
Payload payload = new PayloadProcessor(version).Process(inputStream);
if (payload is null)
{
return(null);
}
ValidateVersion(payload.Version);
DateTime initTime = DateTime.UtcNow;
CommandExecutorStat commandExecutorStat = new CommandExecutor(version).Execute(
inputStream,
outputStream,
payload.SplitIndex,
payload.Command);
DateTime finishTime = DateTime.UtcNow;
WriteDiagnosticsInfo(outputStream, bootTime, initTime, finishTime);
// Mark the beginning of the accumulators section of the output
SerDe.Write(outputStream, (int)SpecialLengths.END_OF_DATA_SECTION);
// TODO: Extend the following to write accumulator values here.
SerDe.Write(outputStream, 0);
// Check the end of stream.
int endOfStream = SerDe.ReadInt32(inputStream);
if (endOfStream == (int)SpecialLengths.END_OF_STREAM)
{
s_logger.LogDebug($"[{TaskId}] Received END_OF_STREAM signal.");
SerDe.Write(outputStream, (int)SpecialLengths.END_OF_STREAM);
readComplete = true;
}
else
{
// This may happen when the input data is not read completely,
// e.g., when take() operation is performed
s_logger.LogWarn($"[{TaskId}] Unexpected end of stream: {endOfStream}.");
s_logger.LogWarn(SerDe.ReadInt32(inputStream).ToString());
// Write a different value to tell JVM to not reuse this worker.
SerDe.Write(outputStream, (int)SpecialLengths.END_OF_DATA_SECTION);
}
LogStat(commandExecutorStat, readComplete);
return(payload);
}
catch (Exception e)
{
s_logger.LogError($"[{TaskId}] ProcessStream() failed with exception: {e}");
try
{
SerDe.Write(outputStream, (int)SpecialLengths.PYTHON_EXCEPTION_THROWN);
SerDe.Write(outputStream, e.ToString());
}
catch (IOException)
{
// JVM closed the socket.
}
catch (Exception ex)
{
s_logger.LogError(
$"[{TaskId}] Writing exception to stream failed with exception: {ex}");
}
throw;
}
}
