internal static Payload GetDefaultPayload()
{
var taskContext = new TaskContext()
{
StageId = 1,
PartitionId = 2,
AttemptNumber = 1,
AttemptId = 100L,
Port = 9999,
Secret = "secret"
};
var broadcastVars = new BroadcastVariables()
{
DecryptionServerNeeded = true,
DecryptionServerPort = 9999,
Secret = "secret"
};
return(new Payload()
{
SplitIndex = 10,
Version = Versions.CurrentVersion,
TaskContext = taskContext,
SparkFilesDir = "directory",
IncludeItems = new[] { "file1", "file2" },
BroadcastVariables = broadcastVars
});
}
/// <summary>
/// Reads the given stream to construct a BroadcastVariables object.
/// </summary>
/// <param name="stream">The stream to read from</param>
/// <returns>BroadcastVariables object</returns>
internal BroadcastVariables Process(Stream stream)
{
var broadcastVars = new BroadcastVariables();
ISocketWrapper socket = null;
broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream);
broadcastVars.Count = Math.Max(SerDe.ReadInt32(stream), 0);
if (broadcastVars.DecryptionServerNeeded)
{
broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream);
broadcastVars.Secret = SerDe.ReadString(stream);
if (broadcastVars.Count > 0)
{
socket = SocketFactory.CreateSocket();
socket.Connect(
IPAddress.Loopback,
broadcastVars.DecryptionServerPort,
broadcastVars.Secret);
}
}
var formatter = new BinaryFormatter();
for (int i = 0; i < broadcastVars.Count; ++i)
{
long bid = SerDe.ReadInt64(stream);
if (bid >= 0)
{
if (broadcastVars.DecryptionServerNeeded)
{
long readBid = SerDe.ReadInt64(socket.InputStream);
if (bid != readBid)
{
throw new Exception("The Broadcast Id received from the encryption " +
$"server {readBid} is different from the Broadcast Id received " +
$"from the payload {bid}.");
}
object value = formatter.Deserialize(socket.InputStream);
BroadcastRegistry.Add(bid, value);
}
else
{
string path = SerDe.ReadString(stream);
using FileStream fStream =
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read);
object value = formatter.Deserialize(fStream);
BroadcastRegistry.Add(bid, value);
}
}
else
{
bid = -bid - 1;
BroadcastRegistry.Remove(bid);
}
}
socket?.Dispose();
return(broadcastVars);
}
public void Write(Stream stream, BroadcastVariables broadcastVars)
{
SerDe.Write(stream, broadcastVars.DecryptionServerNeeded);
SerDe.Write(stream, broadcastVars.Count);
Debug.Assert(broadcastVars.Count == 0);
if (broadcastVars.DecryptionServerNeeded)
{
SerDe.Write(stream, broadcastVars.DecryptionServerPort);
SerDe.Write(stream, broadcastVars.Secret);
}
}
/// <summary>
/// Reads the given stream to construct a BroadcastVariables object.
/// </summary>
/// <param name="stream">The stream to read from</param>
/// <returns>BroadcastVariables object</returns>
internal BroadcastVariables Process(Stream stream)
{
var broadcastVars = new BroadcastVariables();
if (_version >= new Version(Versions.V2_3_2))
{
broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream);
}
broadcastVars.Count = Math.Max(SerDe.ReadInt32(stream), 0);
if (broadcastVars.DecryptionServerNeeded)
{
broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream);
broadcastVars.Secret = SerDe.ReadString(stream);
// TODO: Handle the authentication.
}
var formatter = new BinaryFormatter();
for (int i = 0; i < broadcastVars.Count; ++i)
{
long bid = SerDe.ReadInt64(stream);
if (bid >= 0)
{
if (broadcastVars.DecryptionServerNeeded)
{
throw new NotImplementedException(
"broadcastDecryptionServer is not implemented.");
}
else
{
string path = SerDe.ReadString(stream);
using FileStream fStream =
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read);
object value = formatter.Deserialize(fStream);
BroadcastRegistry.Add(bid, value);
}
}
else
{
bid = -bid - 1;
BroadcastRegistry.Remove(bid);
}
}
return(broadcastVars);
}
/// <summary>
/// Reads the given stream to construct a BroadcastVariables object.
/// </summary>
/// <param name="stream">The stream to read from</param>
/// <returns>BroadcastVariables object</returns>
internal BroadcastVariables Process(Stream stream)
{
var broadcastVars = new BroadcastVariables();
if (_version >= new Version(Versions.V2_3_2))
{
broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream);
}
// Note that broadcast variables are currently ignored.
// Thus, just read the info from stream without handling them.
int numBroadcastVariables = Math.Max(SerDe.ReadInt32(stream), 0);
if (broadcastVars.DecryptionServerNeeded)
{
broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream);
broadcastVars.Secret = SerDe.ReadString(stream);
// TODO: Handle the authentication.
}
for (int i = 0; i < numBroadcastVariables; ++i)
{
long bid = SerDe.ReadInt64(stream);
if (bid >= 0)
{
if (broadcastVars.DecryptionServerNeeded)
{
throw new NotImplementedException(
"broadcastDecryptionServer is not implemented.");
}
else
{
string path = SerDe.ReadString(stream);
// TODO: Register new broadcast variable.
}
}
else
{
bid = -bid - 1;
// TODO: Remove registered broadcast variable.
}
}
return(broadcastVars);
}
public void Write(Stream stream, BroadcastVariables broadcastVars)
{
Debug.Assert(broadcastVars.Count == 0);
SerDe.Write(stream, broadcastVars.Count);
}