internal static Payload GetDefaultPayload() { var taskContext = new TaskContext() { StageId = 1, PartitionId = 2, AttemptNumber = 1, AttemptId = 100L, Port = 9999, Secret = "secret" }; var broadcastVars = new BroadcastVariables() { DecryptionServerNeeded = true, DecryptionServerPort = 9999, Secret = "secret" }; return(new Payload() { SplitIndex = 10, Version = Versions.CurrentVersion, TaskContext = taskContext, SparkFilesDir = "directory", IncludeItems = new[] { "file1", "file2" }, BroadcastVariables = broadcastVars }); }
/// <summary> /// Reads the given stream to construct a BroadcastVariables object. /// </summary> /// <param name="stream">The stream to read from</param> /// <returns>BroadcastVariables object</returns> internal BroadcastVariables Process(Stream stream) { var broadcastVars = new BroadcastVariables(); ISocketWrapper socket = null; broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream); broadcastVars.Count = Math.Max(SerDe.ReadInt32(stream), 0); if (broadcastVars.DecryptionServerNeeded) { broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream); broadcastVars.Secret = SerDe.ReadString(stream); if (broadcastVars.Count > 0) { socket = SocketFactory.CreateSocket(); socket.Connect( IPAddress.Loopback, broadcastVars.DecryptionServerPort, broadcastVars.Secret); } } var formatter = new BinaryFormatter(); for (int i = 0; i < broadcastVars.Count; ++i) { long bid = SerDe.ReadInt64(stream); if (bid >= 0) { if (broadcastVars.DecryptionServerNeeded) { long readBid = SerDe.ReadInt64(socket.InputStream); if (bid != readBid) { throw new Exception("The Broadcast Id received from the encryption " + $"server {readBid} is different from the Broadcast Id received " + $"from the payload {bid}."); } object value = formatter.Deserialize(socket.InputStream); BroadcastRegistry.Add(bid, value); } else { string path = SerDe.ReadString(stream); using FileStream fStream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read); object value = formatter.Deserialize(fStream); BroadcastRegistry.Add(bid, value); } } else { bid = -bid - 1; BroadcastRegistry.Remove(bid); } } socket?.Dispose(); return(broadcastVars); }
public void Write(Stream stream, BroadcastVariables broadcastVars) { SerDe.Write(stream, broadcastVars.DecryptionServerNeeded); SerDe.Write(stream, broadcastVars.Count); Debug.Assert(broadcastVars.Count == 0); if (broadcastVars.DecryptionServerNeeded) { SerDe.Write(stream, broadcastVars.DecryptionServerPort); SerDe.Write(stream, broadcastVars.Secret); } }
/// <summary> /// Reads the given stream to construct a BroadcastVariables object. /// </summary> /// <param name="stream">The stream to read from</param> /// <returns>BroadcastVariables object</returns> internal BroadcastVariables Process(Stream stream) { var broadcastVars = new BroadcastVariables(); if (_version >= new Version(Versions.V2_3_2)) { broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream); } broadcastVars.Count = Math.Max(SerDe.ReadInt32(stream), 0); if (broadcastVars.DecryptionServerNeeded) { broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream); broadcastVars.Secret = SerDe.ReadString(stream); // TODO: Handle the authentication. } var formatter = new BinaryFormatter(); for (int i = 0; i < broadcastVars.Count; ++i) { long bid = SerDe.ReadInt64(stream); if (bid >= 0) { if (broadcastVars.DecryptionServerNeeded) { throw new NotImplementedException( "broadcastDecryptionServer is not implemented."); } else { string path = SerDe.ReadString(stream); using FileStream fStream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read); object value = formatter.Deserialize(fStream); BroadcastRegistry.Add(bid, value); } } else { bid = -bid - 1; BroadcastRegistry.Remove(bid); } } return(broadcastVars); }
/// <summary> /// Reads the given stream to construct a BroadcastVariables object. /// </summary> /// <param name="stream">The stream to read from</param> /// <returns>BroadcastVariables object</returns> internal BroadcastVariables Process(Stream stream) { var broadcastVars = new BroadcastVariables(); if (_version >= new Version(Versions.V2_3_2)) { broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream); } // Note that broadcast variables are currently ignored. // Thus, just read the info from stream without handling them. int numBroadcastVariables = Math.Max(SerDe.ReadInt32(stream), 0); if (broadcastVars.DecryptionServerNeeded) { broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream); broadcastVars.Secret = SerDe.ReadString(stream); // TODO: Handle the authentication. } for (int i = 0; i < numBroadcastVariables; ++i) { long bid = SerDe.ReadInt64(stream); if (bid >= 0) { if (broadcastVars.DecryptionServerNeeded) { throw new NotImplementedException( "broadcastDecryptionServer is not implemented."); } else { string path = SerDe.ReadString(stream); // TODO: Register new broadcast variable. } } else { bid = -bid - 1; // TODO: Remove registered broadcast variable. } } return(broadcastVars); }
public void Write(Stream stream, BroadcastVariables broadcastVars) { Debug.Assert(broadcastVars.Count == 0); SerDe.Write(stream, broadcastVars.Count); }