public void testArrays()
{
Pickler p = new Pickler(false);
byte[] o;
o=p.dumps(new string[] {});
Assert.AreEqual(B(")"), o);
o=p.dumps(new string[] {"abc"});
Assert.AreEqual(B("X\u0003\u0000\u0000\u0000abc\u0085"), o);
o=p.dumps(new string[] {"abc","def"});
Assert.AreEqual(B("X\u0003\u0000\u0000\u0000abcX\u0003\u0000\u0000\u0000def\u0086"), o);
o=p.dumps(new string[] {"abc","def","ghi"});
Assert.AreEqual(B("X\u0003\u0000\u0000\u0000abcX\u0003\u0000\u0000\u0000defX\u0003\u0000\u0000\u0000ghi\u0087"), o);
o=p.dumps(new string[] {"abc","def","ghi","jkl"});
Assert.AreEqual(B("(X\u0003\u0000\u0000\u0000abcX\u0003\u0000\u0000\u0000defX\u0003\u0000\u0000\u0000ghiX\u0003\u0000\u0000\u0000jklt"), o);
o=p.dumps(new char[] {'A','B','C'});
Assert.AreEqual(B("X\u0003\u0000\u0000\u0000ABC"), o);
o=p.dumps(new bool[] {true,false,true});
Assert.AreEqual(B("\u0088\u0089\u0088\u0087"), o);
o=p.dumps(new byte[] {1,2,3});
Assert.AreEqual(B("c__builtin__\nbytearray\nX\u0003\u0000\u0000\u0000\u0001\u0002\u0003X\u0007\u0000\u0000\u0000latin-1\u0086R"), o);
o=p.dumps(new int[] {1,2,3});
Assert.AreEqual(B("carray\narray\nU\u0001i](K\u0001K\u0002K\u0003e\u0086R"), o);
o=p.dumps(new double[] {1.1,2.2,3.3});
Assert.AreEqual(B("carray\narray\nU\u0001d](G?\u00f1\u0099\u0099\u0099\u0099\u0099\u009aG@\u0001\u0099\u0099\u0099\u0099\u0099\u009aG@\nffffffe\u0086R"), o);
}
public void testPickleUnpickleURI()
{
PyroURI uri=new PyroURI("PYRO:test@localhost:9999");
Pickler p=new Pickler();
byte[] pickled_uri=p.dumps(uri);
PyroURI uri2=(PyroURI) U(pickled_uri);
Assert.AreEqual(uri,uri2);
uri=new PyroURI();
pickled_uri=p.dumps(uri);
uri2=(PyroURI) U(pickled_uri);
Assert.AreEqual(uri,uri2);
}
public override byte[] serializeData(object obj)
{
using(var p=new Pickler())
{
return p.dumps(obj);
}
}
public override byte[] serializeCall(string objectId, string method, object[] vargs, IDictionary<string, object> kwargs)
{
using(var p=new Pickler())
{
object[] invokeparams = new object[] {objectId, method, vargs, kwargs};
return p.dumps(invokeparams);
}
}
public void testPickleUnpickleProxy()
{
PyroProxy proxy=new PyroProxy("hostname",9999,"objectid");
Pickler p=new Pickler();
byte[] pickled_proxy=p.dumps(proxy);
object result=U(pickled_proxy);
Assert.IsInstanceOfType(typeof(System.Collections.Hashtable), result); // proxy objects cannot be properly pickled and are pickled as bean, hence HashMap
}
public void testAnonType()
{
var x = new { Name="Harry", Country="UK", Age=34 };
Pickler p = new Pickler();
byte[] ser = p.dumps(x);
Unpickler u = new Unpickler();
object y = u.loads(ser);
IDictionary dict = (IDictionary) y;
Assert.AreEqual(3, dict.Count);
Assert.AreEqual(34, dict["Age"]);
Assert.IsFalse(dict.Contains("__class__"));
}
public void testClass()
{
Pickler p=new Pickler(false);
Unpickler pu=new Unpickler();
byte[] o;
Relative person=new Relative("Tupac",true, new int[] {3,4,5});
o=p.dumps(person);
Hashtable map=(Hashtable) pu.loads(o);
Assert.AreEqual(5, map.Count);
Assert.AreEqual("Pyrolite.Tests.Pickle.PicklerTests+Relative", map["__class__"]);
Assert.AreEqual("Tupac", map["Name"]);
Assert.AreEqual("unspecified", map["Relation"]);
Assert.AreEqual(true, map["Deceased"]);
Assert.AreEqual(new int[] {3,4,5}, map["Values"]);
}
public void testMemoizationRecursiveMemo()
{
byte[] o;
Pickler p=new Pickler();
// self-referencing list
string reused = "reused";
IList list=new ArrayList();
list.Add(reused);
list.Add(reused);
list.Add(list);
o=p.dumps(list);
Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01h\x0000e.", S(o));
Unpickler u = new Unpickler();
ArrayList data = (ArrayList) u.loads(o);
Assert.AreEqual(3, data.Count);
string s1 = (string) data[0];
string s2 = (string) data[1];
ArrayList data2 = (ArrayList) data[2];
Assert.AreEqual("reused", s1);
Assert.AreSame(s1, s2);
Assert.AreSame(data, data2);
Assert.AreSame(data[0], data2[0]);
// self-referencing hashtable
Hashtable h = new Hashtable();
h["myself"] = h;
o=p.dumps(h);
Assert.AreEqual("\x80\x02}q\x00(X\x06\x00\x00\x0000myselfq\x01h\x0000u.", S(o));
Hashtable h2 = (Hashtable) u.loads(o);
Assert.AreEqual(1, h2.Count);
Assert.AreSame(h2, h2["myself"]);
}
public void testMemoizationRecursiveNoMemo()
{
byte[] o;
Pickler p=new Pickler(false);
string reused = "reused";
IList list=new ArrayList();
list.Add(reused);
list.Add(reused);
list.Add(list);
o=p.dumps(list);
}
public void testMemoization()
{
byte[] o;
Pickler p=new Pickler();
string reused = "reused";
string another = "another";
IList list=new ArrayList();
IList sublist = new ArrayList();
sublist.Add(reused);
sublist.Add(reused);
sublist.Add(another);
list.Add(reused);
list.Add(reused);
list.Add(another);
list.Add(sublist);
o=p.dumps(list);
Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01X\x07\x00\x00\x0000anotherq\x02]q\x03(h\x01h\x01h\x0002ee.", S(o));
Unpickler u = new Unpickler();
ArrayList data = (ArrayList) u.loads(o);
Assert.AreEqual(4, data.Count);
string s1 = (string) data[0];
string s2 = (string) data[1];
string s3 = (string) data[2];
data = (ArrayList) data[3];
string s4 = (string) data[0];
string s5 = (string) data[1];
string s6 = (string) data[2];
Assert.AreEqual("reused", s1);
Assert.AreEqual("another", s3);
Assert.AreSame(s1, s2);
Assert.AreSame(s3, s6);
Assert.AreSame(s1, s4);
Assert.AreSame(s1, s5);
}
public void testMemoizationTimeStuff()
{
TimeSpan delta = new TimeSpan(1,2,3);
DateTime time = new DateTime(2014,11,20,1,2,3);
object[] array = new object[] {delta, delta, time, time};
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[]) u.loads(data);
Assert.AreEqual(4, result.Length);
Assert.IsInstanceOf(typeof(TimeSpan), result[0]);
Assert.IsInstanceOf(typeof(TimeSpan), result[1]);
Assert.IsInstanceOf(typeof(DateTime), result[2]);
Assert.IsInstanceOf(typeof(DateTime), result[3]);
Assert.AreSame(result[0], result[1]); // both objects should be the same memoized object
Assert.AreSame(result[2], result[3]); // both objects should be the same memoized object
delta = (TimeSpan) result[1];
time = (DateTime) result[3];
Assert.AreEqual(new TimeSpan(1,2,3), delta);
Assert.AreEqual(new DateTime(2014,11,20,1,2,3), time);
}
public void testSinglePrimitives() {
// protocol level 2
Pickler p=new Pickler(false);
byte[] o=p.dumps(null); // none
Assert.AreEqual(B("N"), o);
o=p.dumps('@'); // char --> string
Assert.AreEqual(B("X\u0001\u0000\u0000\u0000@"), o);
o=p.dumps(true); // bool
Assert.AreEqual(B("\u0088"), o);
o=p.dumps("hello"); // unicode string
Assert.AreEqual(B("X\u0005\u0000\u0000\u0000hello"), o);
o=p.dumps("hello\u20ac"); // unicode string with non ascii
Assert.AreEqual(B("X\u0008\u0000\u0000\u0000hello\u00e2\u0082\u00ac"), o);
o=p.dumps((byte)'@');
Assert.AreEqual(B("K@"), o);
o=p.dumps((sbyte)-40);
Assert.AreEqual( B(new byte[]{(byte)'J',0xd8,0xff,0xff,0xff}), o);
o=p.dumps((short)-0x1234);
Assert.AreEqual(B("J\u00cc\u00ed\u00ff\u00ff"), o);
o=p.dumps((ushort)0xf234);
Assert.AreEqual(B("M\u0034\u00f2"), o);
o=p.dumps((int)-0x12345678);
Assert.AreEqual(B("J\u0088\u00a9\u00cb\u00ed"), o);
o=p.dumps((uint)0x12345678);
Assert.AreEqual(B(new byte[]{(byte)'J', 0x78, 0x56, 0x34, 0x12}), o);
o=p.dumps((uint)0xf2345678);
Assert.AreEqual(B("I4063516280\n"), o);
o=p.dumps((long)0x12345678abcdefL);
Assert.AreEqual(B("I5124095577148911\n"), o);
o=p.dumps(1234.5678d);
Assert.AreEqual(B(new byte[] {(byte)'G',0x40,0x93,0x4a,0x45,0x6d,0x5c,0xfa,0xad}), o);
o=p.dumps(1234.5f);
Assert.AreEqual(B(new byte[] {(byte)'G',0x40,0x93,0x4a,0,0,0,0,0}), o);
o=p.dumps(1234.9876543210987654321m);
Assert.AreEqual(B("cdecimal\nDecimal\nX\u0018\u0000\u0000\u00001234.9876543210987654321\u0085R"), o);
DayEnum day=DayEnum.WEDNESDAY;
o=p.dumps(day); // enum is returned as just a string representing the value
Assert.AreEqual(B("X\u0009\u0000\u0000\u0000WEDNESDAY"),o);
}
public void testMemoizationSet()
{
var set = new HashSet<string>();
set.Add("a");
object[] array = new object[] {set, set};
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[]) u.loads(data);
Assert.AreEqual(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsInstanceOf(typeof(HashSet<object>), first);
Assert.IsInstanceOf(typeof(HashSet<object>), second);
Assert.AreSame(first, second); // both objects should be the same memoized object
HashSet<object> theSet = (HashSet<object>)second;
Assert.AreEqual(1, theSet.Count);
Assert.IsTrue(theSet.Contains("a"));
}
public void testDates()
{
Pickler p=new Pickler(false);
Unpickler u=new Unpickler();
DateTime date=new DateTime(2011,12,31,14,33,59);
byte[] o = p.dumps(date);
Object unpickled=u.loads(o);
Assert.AreEqual(date, unpickled);
date=new DateTime(2011,12,31,14,33,59,456);
o=p.dumps(date);
unpickled=u.loads(o);
Assert.AreEqual(date, unpickled);
}
public void TestRecursiveArray6()
{
Pickler p = new Pickler(false);
object[] a = new object[] { "a","b","c","d","e","f" };
a[5] = a; // make it recursive
p.dumps(a);
}
public void TestRecursiveArray2()
{
Pickler p = new Pickler(false);
object[] a = new object[] { "hello", "placeholder" };
a[1] = a; // make it recursive
p.dumps(a);
}
static void Main(string[] args)
{
PrintFiles();
int javaPort = int.Parse(Console.ReadLine());
Log("java_port: " + javaPort);
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sock.Connect(IPAddress.Parse("127.0.0.1"), javaPort);
using (NetworkStream s = new NetworkStream(sock))
{
try
{
DateTime bootTime = DateTime.UtcNow;
int splitIndex = SerDe.ReadInt(s);
Log("split_index: " + splitIndex);
if (splitIndex == -1)
Environment.Exit(-1);
string ver = SerDe.ReadString(s);
Log("ver: " + ver);
//// initialize global state
//shuffle.MemoryBytesSpilled = 0
//shuffle.DiskBytesSpilled = 0
// fetch name of workdir
string sparkFilesDir = SerDe.ReadString(s);
Log("spark_files_dir: " + sparkFilesDir);
//SparkFiles._root_directory = sparkFilesDir
//SparkFiles._is_running_on_worker = True
// fetch names of includes - not used //TODO - complete the impl
int numberOfIncludesItems = SerDe.ReadInt(s);
Log("num_includes: " + numberOfIncludesItems);
if (numberOfIncludesItems > 0)
{
for (int i = 0; i < numberOfIncludesItems; i++)
{
string filename = SerDe.ReadString(s);
}
}
// fetch names and values of broadcast variables
int numBroadcastVariables = SerDe.ReadInt(s);
Log("num_broadcast_variables: " + numBroadcastVariables);
if (numBroadcastVariables > 0)
{
for (int i = 0; i < numBroadcastVariables; i++)
{
long bid = SerDe.ReadLong(s);
if (bid >= 0)
{
string path = SerDe.ReadString(s);
Broadcast.broadcastRegistry[bid] = new Broadcast(path);
}
else
{
bid = -bid - 1;
Broadcast.broadcastRegistry.Remove(bid);
}
}
}
Accumulator.accumulatorRegistry.Clear();
int lengthOCommandByteArray = SerDe.ReadInt(s);
Log("command_len: " + lengthOCommandByteArray);
IFormatter formatter = new BinaryFormatter();
if (lengthOCommandByteArray > 0)
{
string deserializerMode = SerDe.ReadString(s);
Log("Deserializer mode: " + deserializerMode);
string serializerMode = SerDe.ReadString(s);
Log("Serializer mode: " + serializerMode);
byte[] command = SerDe.ReadBytes(s);
Log("command bytes read: " + command.Length);
var stream = new MemoryStream(command);
var func = (Func<int, IEnumerable<dynamic>, IEnumerable<dynamic>>)formatter.Deserialize(stream);
DateTime initTime = DateTime.UtcNow;
int count = 0;
foreach (var message in func(splitIndex, GetIterator(s, deserializerMode)))
{
byte[] buffer;
if (serializerMode == "None")
{
buffer = message as byte[];
}
else if (serializerMode == "String")
{
buffer = SerDe.ToBytes(message as string);
}
else if (serializerMode == "Row")
{
Pickler pickler = new Pickler();
buffer = pickler.dumps(new ArrayList { message });
}
else
{
try
{
var ms = new MemoryStream();
formatter.Serialize(ms, message);
buffer = ms.ToArray();
}
catch (Exception)
{
Log(string.Format("{0} : {1}", message.GetType().Name, message.GetType().FullName));
throw;
}
}
count++;
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
//TODO - complete the impl
Log("Count: " + count);
//if profiler:
// profiler.profile(process)
//else:
// process()
DateTime finish_time = DateTime.UtcNow;
SerDe.Write(s, (int)SpecialLengths.TIMING_DATA);
SerDe.Write(s, ToUnixTime(bootTime));
SerDe.Write(s, ToUnixTime(initTime));
SerDe.Write(s, ToUnixTime(finish_time));
SerDe.Write(s, 0L); //shuffle.MemoryBytesSpilled
SerDe.Write(s, 0L); //shuffle.DiskBytesSpilled
}
else
{
Log("Nothing to execute :-(");
}
// Mark the beginning of the accumulators section of the output
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, Accumulator.accumulatorRegistry.Count);
foreach (var item in Accumulator.accumulatorRegistry)
{
var ms = new MemoryStream();
var value = item.Value.GetType().GetField("value", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(item.Value);
Log(string.Format("({0}, {1})", item.Key, value));
formatter.Serialize(ms, new KeyValuePair<int, dynamic>(item.Key, value));
byte[] buffer = ms.ToArray();
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
int end = SerDe.ReadInt(s);
// check end of stream
if (end == (int)SpecialLengths.END_OF_DATA_SECTION || end == (int)SpecialLengths.END_OF_STREAM)
{
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
Log("END_OF_STREAM: " + (int)SpecialLengths.END_OF_STREAM);
}
else
{
// write a different value to tell JVM to not reuse this worker
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
Environment.Exit(-1);
}
s.Flush();
System.Threading.Thread.Sleep(1000); //TODO - not sure if this is really needed
}
catch (Exception e)
{
Log(e.ToString());
try
{
SerDe.Write(s, e.ToString());
}
catch (IOException)
{
// JVM close the socket
}
catch (Exception ex)
{
LogError("CSharpWorker failed with exception:");
LogError(ex.ToString());
}
Environment.Exit(-1);
}
}
}
static void Main(string[] args)
{
// if there exists exe.config file, then use log4net
if (File.Exists(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile))
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance);
}
logger = LoggerServiceFactory.GetLogger(typeof(Worker));
Socket sock = null;
try
{
PrintFiles();
int javaPort = int.Parse(Console.ReadLine());
logger.LogInfo("java_port: " + javaPort);
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sock.Connect(IPAddress.Loopback, javaPort);
}
catch (Exception e)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(e);
Environment.Exit(-1);
}
using (NetworkStream s = new NetworkStream(sock))
{
try
{
DateTime bootTime = DateTime.UtcNow;
int splitIndex = SerDe.ReadInt(s);
logger.LogInfo("split_index: " + splitIndex);
if (splitIndex == -1)
Environment.Exit(-1);
string ver = SerDe.ReadString(s);
logger.LogInfo("ver: " + ver);
//// initialize global state
//shuffle.MemoryBytesSpilled = 0
//shuffle.DiskBytesSpilled = 0
// fetch name of workdir
string sparkFilesDir = SerDe.ReadString(s);
logger.LogInfo("spark_files_dir: " + sparkFilesDir);
//SparkFiles._root_directory = sparkFilesDir
//SparkFiles._is_running_on_worker = True
// fetch names of includes - not used //TODO - complete the impl
int numberOfIncludesItems = SerDe.ReadInt(s);
logger.LogInfo("num_includes: " + numberOfIncludesItems);
if (numberOfIncludesItems > 0)
{
for (int i = 0; i < numberOfIncludesItems; i++)
{
string filename = SerDe.ReadString(s);
}
}
// fetch names and values of broadcast variables
int numBroadcastVariables = SerDe.ReadInt(s);
logger.LogInfo("num_broadcast_variables: " + numBroadcastVariables);
if (numBroadcastVariables > 0)
{
for (int i = 0; i < numBroadcastVariables; i++)
{
long bid = SerDe.ReadLong(s);
if (bid >= 0)
{
string path = SerDe.ReadString(s);
Broadcast.broadcastRegistry[bid] = new Broadcast(path);
}
else
{
bid = -bid - 1;
Broadcast.broadcastRegistry.Remove(bid);
}
}
}
Accumulator.accumulatorRegistry.Clear();
int lengthOCommandByteArray = SerDe.ReadInt(s);
logger.LogInfo("command_len: " + lengthOCommandByteArray);
IFormatter formatter = new BinaryFormatter();
if (lengthOCommandByteArray > 0)
{
Stopwatch commandProcessWatch = new Stopwatch();
Stopwatch funcProcessWatch = new Stopwatch();
commandProcessWatch.Start();
int rddId = SerDe.ReadInt(s);
int stageId = SerDe.ReadInt(s);
int partitionId = SerDe.ReadInt(s);
logger.LogInfo(string.Format("rddInfo: rddId {0}, stageId {1}, partitionId {2}", rddId, stageId, partitionId));
string deserializerMode = SerDe.ReadString(s);
logger.LogInfo("Deserializer mode: " + deserializerMode);
string serializerMode = SerDe.ReadString(s);
logger.LogInfo("Serializer mode: " + serializerMode);
byte[] command = SerDe.ReadBytes(s);
logger.LogInfo("command bytes read: " + command.Length);
var stream = new MemoryStream(command);
var workerFunc = (CSharpWorkerFunc)formatter.Deserialize(stream);
var func = workerFunc.Func;
logger.LogInfo(string.Format("stack trace of workerFunc (dont't panic, this is just for debug):\n{0}", workerFunc.StackTrace));
DateTime initTime = DateTime.UtcNow;
int count = 0;
// here we use low level API because we need to get perf metrics
WorkerInputEnumerator inputEnumerator = new WorkerInputEnumerator(s, deserializerMode);
IEnumerable<dynamic> inputEnumerable = Enumerable.Cast<dynamic>(inputEnumerator);
funcProcessWatch.Start();
IEnumerable<dynamic> outputEnumerable = func(splitIndex, inputEnumerable);
var outputEnumerator = outputEnumerable.GetEnumerator();
funcProcessWatch.Stop();
while (true)
{
funcProcessWatch.Start();
bool hasNext = outputEnumerator.MoveNext();
funcProcessWatch.Stop();
if (!hasNext)
{
break;
}
funcProcessWatch.Start();
var message = outputEnumerator.Current;
funcProcessWatch.Stop();
if (object.ReferenceEquals(null, message))
{
continue;
}
byte[] buffer;
if (serializerMode == "None")
{
buffer = message as byte[];
}
else if (serializerMode == "String")
{
buffer = SerDe.ToBytes(message as string);
}
else if (serializerMode == "Row")
{
Pickler pickler = new Pickler();
buffer = pickler.dumps(new ArrayList { message });
}
else
{
try
{
var ms = new MemoryStream();
formatter.Serialize(ms, message);
buffer = ms.ToArray();
}
catch (Exception)
{
logger.LogError(string.Format("{0} : {1}", message.GetType().Name, message.GetType().FullName));
throw;
}
}
count++;
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
//TODO - complete the impl
logger.LogInfo("Count: " + count);
//if profiler:
// profiler.profile(process)
//else:
// process()
DateTime finish_time = DateTime.UtcNow;
const string format = "MM/dd/yyyy hh:mm:ss.fff tt";
logger.LogInfo(string.Format("bootTime: {0}, initTime: {1}, finish_time: {2}",
bootTime.ToString(format), initTime.ToString(format), finish_time.ToString(format)));
SerDe.Write(s, (int)SpecialLengths.TIMING_DATA);
SerDe.Write(s, ToUnixTime(bootTime));
SerDe.Write(s, ToUnixTime(initTime));
SerDe.Write(s, ToUnixTime(finish_time));
SerDe.Write(s, 0L); //shuffle.MemoryBytesSpilled
SerDe.Write(s, 0L); //shuffle.DiskBytesSpilled
commandProcessWatch.Stop();
// log statistics
inputEnumerator.LogStatistic();
logger.LogInfo(string.Format("func process time: {0}", funcProcessWatch.ElapsedMilliseconds));
logger.LogInfo(string.Format("command process time: {0}", commandProcessWatch.ElapsedMilliseconds));
}
else
{
logger.LogWarn("Nothing to execute :-(");
}
// Mark the beginning of the accumulators section of the output
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, Accumulator.accumulatorRegistry.Count);
foreach (var item in Accumulator.accumulatorRegistry)
{
var ms = new MemoryStream();
var value = item.Value.GetType().GetField("value", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(item.Value);
logger.LogInfo(string.Format("({0}, {1})", item.Key, value));
formatter.Serialize(ms, new KeyValuePair<int, dynamic>(item.Key, value));
byte[] buffer = ms.ToArray();
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
int end = SerDe.ReadInt(s);
// check end of stream
if (end == (int)SpecialLengths.END_OF_STREAM)
{
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
logger.LogInfo("END_OF_STREAM: " + (int)SpecialLengths.END_OF_STREAM);
}
else
{
// write a different value to tell JVM to not reuse this worker
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
Environment.Exit(-1);
}
s.Flush();
// log bytes read and write
logger.LogInfo(string.Format("total read bytes: {0}", SerDe.totalReadNum));
logger.LogInfo(string.Format("total write bytes: {0}", SerDe.totalWriteNum));
// wait for server to complete, otherwise server gets 'connection reset' exception
// Use SerDe.ReadBytes() to detect java side has closed socket properly
// ReadBytes() will block until the socket is closed
SerDe.ReadBytes(s);
}
catch (Exception e)
{
logger.LogError(e.ToString());
try
{
SerDe.Write(s, e.ToString());
}
catch (IOException)
{
// JVM close the socket
}
catch (Exception ex)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(ex);
}
Environment.Exit(-1);
}
}
sock.Close();
}
public void testFailure()
{
NotABean notabean=new NotABean();
notabean.x=42;
Pickler p=new Pickler(false);
p.dumps(notabean);
}
public void testDateTimeStringEscaping()
{
DateTime dt=new DateTime(2011, 10, 10, 9, 13, 10, 10);
Pickler p = new Pickler();
byte[] pickle = p.dumps(dt);
Unpickler u = new Unpickler();
DateTime dt2 = (DateTime) u.loads(pickle);
Assert.AreEqual(dt, dt2);
dt = new DateTime(2011, 10, 9, 13, 10, 9, 10);
dt2 = (DateTime) U("\u0080\u0002cdatetime\ndatetime\nq\u0000U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001\u0085q\u0002Rq\u0003."); // protocol 2
Assert.AreEqual(dt, dt2);
dt2 = (DateTime) U("cdatetime\ndatetime\nq\u0000(U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001tq\u0002Rq\u0003."); // protocol 1
Assert.AreEqual(dt, dt2);
dt2 = (DateTime) U("cdatetime\ndatetime\np0\n(S'\\x07\\xdb\\n\\t\\r\\n\\t\\x00'\\x10'\np1\ntp2\nRp3\n."); // protocol 0
Assert.AreEqual(dt, dt2);
}
public void testCustomPickler()
{
Pickler.registerCustomPickler(typeof(CustomClass), new CustomClassPickler());
CustomClass c=new CustomClass();
Pickler p=new Pickler(false);
byte[] ser = p.dumps(c);
Unpickler u = new Unpickler();
string x = (string) u.loads(ser);
Assert.AreEqual("customclassint=42", x);
}
public void testZeroToTwoFiveSix()
{
byte[] bytes=new byte[256];
for(int b=0; b<256; ++b) {
bytes[b]=(byte)b;
}
StringBuilder sb=new StringBuilder();
for(int i=0; i<256; ++i) {
sb.Append((char)i);
}
string str=sb.ToString();
Pickler p=new Pickler(false);
Unpickler u=new Unpickler();
MemoryStream bos=new MemoryStream(434);
bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2);
byte[] data=Encoding.Default.GetBytes("c__builtin__\nbytearray\n");
bos.Write(data,0,data.Length);
bos.WriteByte(Opcodes.BINUNICODE);
bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4);
byte[] utf8=Encoding.UTF8.GetBytes(str);
bos.Write(utf8,0,utf8.Length);
bos.WriteByte(Opcodes.BINUNICODE);
bos.Write(new byte[] {7,0,0,0},0,4);
data=Encoding.Default.GetBytes("latin-1");
bos.Write(data,0,data.Length);
bos.WriteByte(Opcodes.TUPLE2);
bos.WriteByte(Opcodes.REDUCE);
bos.WriteByte(Opcodes.STOP);
byte[] bytesresult=bos.ToArray();
byte[] output=p.dumps(bytes);
Assert.AreEqual(bytesresult, output);
Assert.AreEqual(bytes, (byte[])u.loads(output));
bos=new MemoryStream(434);
bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2);
bos.WriteByte(Opcodes.BINUNICODE);
bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4);
utf8=Encoding.UTF8.GetBytes(str);
bos.Write(utf8,0,utf8.Length);
bos.WriteByte(Opcodes.STOP);
bytesresult=bos.ToArray();
output=p.dumps(str);
Assert.AreEqual(bytesresult, output);
Assert.AreEqual(str, u.loads(output));
}
public void testTimes()
{
Pickler p=new Pickler(false);
Unpickler u=new Unpickler();
TimeSpan ts=new TimeSpan(2, 0, 0, 7000, 456);
byte[] o = p.dumps(ts);
object unpickled=u.loads(o);
Assert.AreEqual(ts,unpickled);
Assert.AreEqual(B("cdatetime\ntimedelta\nK\u0002MX\u001bJ@\u00f5\u0006\u0000\u0087R"), o);
}
public void testMemoizationString()
{
string str = "a";
object[] array = new object[] {str, str};
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[]) u.loads(data);
Assert.AreEqual(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsInstanceOf(typeof(string), first);
Assert.IsInstanceOf(typeof(string), second);
Assert.AreSame(first, second); // both objects should be the same memoized object
str = (string) second;
Assert.AreEqual("a", str);
}
private static byte[] GetSerializedMessage(string serializerMode, dynamic message, IFormatter formatter)
{
byte[] buffer;
switch ((SerializedMode)Enum.Parse(typeof(SerializedMode), serializerMode))
{
case SerializedMode.None:
buffer = message as byte[];
break;
case SerializedMode.String:
buffer = SerDe.ToBytes(message as string);
break;
case SerializedMode.Row:
var pickler = new Pickler();
buffer = pickler.dumps(new ArrayList { message });
break;
default:
try
{
var ms = new MemoryStream();
formatter.Serialize(ms, message);
buffer = ms.ToArray();
}
catch (Exception)
{
logger.LogError("Exception serializing output");
logger.LogError("{0} : {1}", message.GetType().Name, message.GetType().FullName);
throw;
}
break;
}
return buffer;
}
public void testSets()
{
byte[] o;
Pickler p=new Pickler(false);
Unpickler up=new Unpickler();
var intset=new HashSet<int>();
intset.Add(1);
intset.Add(2);
intset.Add(3);
o=p.dumps(intset);
HashSet<object> resultset=(HashSet<object>)up.loads(o);
AssertUtils.AssertEqual(intset, resultset);
HashSet<string> stringset=new HashSet<string>();
stringset.Add("A");
stringset.Add("B");
stringset.Add("C");
o=p.dumps(stringset);
resultset=(HashSet<object>)up.loads(o);
AssertUtils.AssertEqual(stringset, resultset);
}
public void testUnpicklingPerformance()
{
Pickler pickler = new Pickler();
var myList = new List<string>();
for (int i = 0; i < 10; i++) {
myList.Add(i.ToString());
}
byte[] bytes = pickler.dumps(myList);
Unpickler unpickler = new Unpickler();
DateTime start = DateTime.Now;
for (int i = 0; i < 1000000; i++) {
unpickler.loads(bytes);
}
Console.WriteLine(DateTime.Now - start);
}
public void testMappings()
{
byte[] o;
Pickler p=new Pickler(false);
Unpickler pu=new Unpickler();
var intmap=new Dictionary<int,int>();
intmap.Add(1, 11);
intmap.Add(2, 22);
intmap.Add(3, 33);
o=p.dumps(intmap);
Hashtable resultmap=(Hashtable)pu.loads(o);
AssertUtils.AssertEqual(intmap, resultmap);
var stringmap=new Dictionary<string,string>();
stringmap.Add("A", "1");
stringmap.Add("B", "2");
stringmap.Add("C", "3");
o=p.dumps(stringmap);
resultmap=(Hashtable)pu.loads(o);
AssertUtils.AssertEqual(stringmap, resultmap);
Hashtable table=new Hashtable();
table.Add(1,11);
table.Add(2,22);
table.Add(3,33);
o=p.dumps(table);
resultmap=(Hashtable)pu.loads(o);
AssertUtils.AssertEqual(table, resultmap);
}
public void TestWorkerWithRowDeserializedModeAndBytesSerializedMode()
{
Process worker;
var CSharpRDD_SocketServer = CreateServer(out worker);
const int expectedCount = 5;
using (var serverSocket = CSharpRDD_SocketServer.Accept())
using (var s = serverSocket.GetStream())
{
WritePayloadHeaderToWorker(s);
byte[] commandWithRowDeserializeMode =
SparkContext.BuildCommand(new CSharpWorkerFunc((pid, iter) => iter), SerializedMode.Row);
SerDe.Write(s, commandWithRowDeserializeMode.Length);
SerDe.Write(s, commandWithRowDeserializeMode);
new StructTypePickler().Register();
new RowPickler().Register();
Pickler pickler = new Pickler();
for (int i = 0; i < expectedCount; i++)
{
byte[] pickleBytes = pickler.dumps(new[] { RowHelper.BuildRowForBasicSchema(i) });
SerDe.Write(s, pickleBytes.Length);
SerDe.Write(s, pickleBytes);
}
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
s.Flush();
int count = 0;
var formatter = new BinaryFormatter();
foreach (var bytes in ReadWorker(s))
{
var ms = new MemoryStream(bytes);
var rows = new ArrayList { formatter.Deserialize(ms) }.Cast<Row>().ToArray();
Assert.AreEqual(1, rows.Count());
Assert.AreEqual(count, rows[0].Get("age"));
count++;
}
Assert.AreEqual(expectedCount, count);
}
AssertWorker(worker);
CSharpRDD_SocketServer.Close();
}
public void testLists()
{
byte[] o;
Pickler p=new Pickler(false);
IList list=new ArrayList();
list.Add(1);
list.Add("abc");
list.Add(null);
o=p.dumps(list);
Assert.AreEqual(B("](K\u0001X\u0003\u0000\u0000\u0000abcNe"), o);
IList<object> ilist=new List<object>();
ilist.Add(1);
ilist.Add("abc");
ilist.Add(null);
o=p.dumps(ilist);
Assert.AreEqual(B("](K\u0001X\u0003\u0000\u0000\u0000abcNe"), o);
Stack<int> stack=new Stack<int>();
stack.Push(1);
stack.Push(2);
stack.Push(3);
o=p.dumps(stack);
Assert.AreEqual(B("](K\u0003K\u0002K\u0001e"), o);
var queue=new Queue<int>();
queue.Enqueue(1);
queue.Enqueue(2);
queue.Enqueue(3);
o=p.dumps(queue);
Assert.AreEqual(B("](K\u0001K\u0002K\u0003e"), o);
}
