public void testArrays() { Pickler p = new Pickler(false); byte[] o; o=p.dumps(new string[] {}); Assert.AreEqual(B(")"), o); o=p.dumps(new string[] {"abc"}); Assert.AreEqual(B("X\u0003\u0000\u0000\u0000abc\u0085"), o); o=p.dumps(new string[] {"abc","def"}); Assert.AreEqual(B("X\u0003\u0000\u0000\u0000abcX\u0003\u0000\u0000\u0000def\u0086"), o); o=p.dumps(new string[] {"abc","def","ghi"}); Assert.AreEqual(B("X\u0003\u0000\u0000\u0000abcX\u0003\u0000\u0000\u0000defX\u0003\u0000\u0000\u0000ghi\u0087"), o); o=p.dumps(new string[] {"abc","def","ghi","jkl"}); Assert.AreEqual(B("(X\u0003\u0000\u0000\u0000abcX\u0003\u0000\u0000\u0000defX\u0003\u0000\u0000\u0000ghiX\u0003\u0000\u0000\u0000jklt"), o); o=p.dumps(new char[] {'A','B','C'}); Assert.AreEqual(B("X\u0003\u0000\u0000\u0000ABC"), o); o=p.dumps(new bool[] {true,false,true}); Assert.AreEqual(B("\u0088\u0089\u0088\u0087"), o); o=p.dumps(new byte[] {1,2,3}); Assert.AreEqual(B("c__builtin__\nbytearray\nX\u0003\u0000\u0000\u0000\u0001\u0002\u0003X\u0007\u0000\u0000\u0000latin-1\u0086R"), o); o=p.dumps(new int[] {1,2,3}); Assert.AreEqual(B("carray\narray\nU\u0001i](K\u0001K\u0002K\u0003e\u0086R"), o); o=p.dumps(new double[] {1.1,2.2,3.3}); Assert.AreEqual(B("carray\narray\nU\u0001d](G?\u00f1\u0099\u0099\u0099\u0099\u0099\u009aG@\u0001\u0099\u0099\u0099\u0099\u0099\u009aG@\nffffffe\u0086R"), o); }
public void testPickleUnpickleURI() { PyroURI uri=new PyroURI("PYRO:test@localhost:9999"); Pickler p=new Pickler(); byte[] pickled_uri=p.dumps(uri); PyroURI uri2=(PyroURI) U(pickled_uri); Assert.AreEqual(uri,uri2); uri=new PyroURI(); pickled_uri=p.dumps(uri); uri2=(PyroURI) U(pickled_uri); Assert.AreEqual(uri,uri2); }
public override byte[] serializeData(object obj) { using(var p=new Pickler()) { return p.dumps(obj); } }
public override byte[] serializeCall(string objectId, string method, object[] vargs, IDictionary<string, object> kwargs) { using(var p=new Pickler()) { object[] invokeparams = new object[] {objectId, method, vargs, kwargs}; return p.dumps(invokeparams); } }
public void testPickleUnpickleProxy() { PyroProxy proxy=new PyroProxy("hostname",9999,"objectid"); Pickler p=new Pickler(); byte[] pickled_proxy=p.dumps(proxy); object result=U(pickled_proxy); Assert.IsInstanceOfType(typeof(System.Collections.Hashtable), result); // proxy objects cannot be properly pickled and are pickled as bean, hence HashMap }
public void testAnonType() { var x = new { Name="Harry", Country="UK", Age=34 }; Pickler p = new Pickler(); byte[] ser = p.dumps(x); Unpickler u = new Unpickler(); object y = u.loads(ser); IDictionary dict = (IDictionary) y; Assert.AreEqual(3, dict.Count); Assert.AreEqual(34, dict["Age"]); Assert.IsFalse(dict.Contains("__class__")); }
public void testClass() { Pickler p=new Pickler(false); Unpickler pu=new Unpickler(); byte[] o; Relative person=new Relative("Tupac",true, new int[] {3,4,5}); o=p.dumps(person); Hashtable map=(Hashtable) pu.loads(o); Assert.AreEqual(5, map.Count); Assert.AreEqual("Pyrolite.Tests.Pickle.PicklerTests+Relative", map["__class__"]); Assert.AreEqual("Tupac", map["Name"]); Assert.AreEqual("unspecified", map["Relation"]); Assert.AreEqual(true, map["Deceased"]); Assert.AreEqual(new int[] {3,4,5}, map["Values"]); }
public void testMemoizationRecursiveMemo() { byte[] o; Pickler p=new Pickler(); // self-referencing list string reused = "reused"; IList list=new ArrayList(); list.Add(reused); list.Add(reused); list.Add(list); o=p.dumps(list); Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01h\x0000e.", S(o)); Unpickler u = new Unpickler(); ArrayList data = (ArrayList) u.loads(o); Assert.AreEqual(3, data.Count); string s1 = (string) data[0]; string s2 = (string) data[1]; ArrayList data2 = (ArrayList) data[2]; Assert.AreEqual("reused", s1); Assert.AreSame(s1, s2); Assert.AreSame(data, data2); Assert.AreSame(data[0], data2[0]); // self-referencing hashtable Hashtable h = new Hashtable(); h["myself"] = h; o=p.dumps(h); Assert.AreEqual("\x80\x02}q\x00(X\x06\x00\x00\x0000myselfq\x01h\x0000u.", S(o)); Hashtable h2 = (Hashtable) u.loads(o); Assert.AreEqual(1, h2.Count); Assert.AreSame(h2, h2["myself"]); }
public void testMemoizationRecursiveNoMemo() { byte[] o; Pickler p=new Pickler(false); string reused = "reused"; IList list=new ArrayList(); list.Add(reused); list.Add(reused); list.Add(list); o=p.dumps(list); }
public void testMemoization() { byte[] o; Pickler p=new Pickler(); string reused = "reused"; string another = "another"; IList list=new ArrayList(); IList sublist = new ArrayList(); sublist.Add(reused); sublist.Add(reused); sublist.Add(another); list.Add(reused); list.Add(reused); list.Add(another); list.Add(sublist); o=p.dumps(list); Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01X\x07\x00\x00\x0000anotherq\x02]q\x03(h\x01h\x01h\x0002ee.", S(o)); Unpickler u = new Unpickler(); ArrayList data = (ArrayList) u.loads(o); Assert.AreEqual(4, data.Count); string s1 = (string) data[0]; string s2 = (string) data[1]; string s3 = (string) data[2]; data = (ArrayList) data[3]; string s4 = (string) data[0]; string s5 = (string) data[1]; string s6 = (string) data[2]; Assert.AreEqual("reused", s1); Assert.AreEqual("another", s3); Assert.AreSame(s1, s2); Assert.AreSame(s3, s6); Assert.AreSame(s1, s4); Assert.AreSame(s1, s5); }
public void testMemoizationTimeStuff() { TimeSpan delta = new TimeSpan(1,2,3); DateTime time = new DateTime(2014,11,20,1,2,3); object[] array = new object[] {delta, delta, time, time}; Pickler p = new Pickler(true); byte[] data = p.dumps(array); Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done Unpickler u = new Unpickler(); object[] result = (object[]) u.loads(data); Assert.AreEqual(4, result.Length); Assert.IsInstanceOf(typeof(TimeSpan), result[0]); Assert.IsInstanceOf(typeof(TimeSpan), result[1]); Assert.IsInstanceOf(typeof(DateTime), result[2]); Assert.IsInstanceOf(typeof(DateTime), result[3]); Assert.AreSame(result[0], result[1]); // both objects should be the same memoized object Assert.AreSame(result[2], result[3]); // both objects should be the same memoized object delta = (TimeSpan) result[1]; time = (DateTime) result[3]; Assert.AreEqual(new TimeSpan(1,2,3), delta); Assert.AreEqual(new DateTime(2014,11,20,1,2,3), time); }
public void testSinglePrimitives() { // protocol level 2 Pickler p=new Pickler(false); byte[] o=p.dumps(null); // none Assert.AreEqual(B("N"), o); o=p.dumps('@'); // char --> string Assert.AreEqual(B("X\u0001\u0000\u0000\u0000@"), o); o=p.dumps(true); // bool Assert.AreEqual(B("\u0088"), o); o=p.dumps("hello"); // unicode string Assert.AreEqual(B("X\u0005\u0000\u0000\u0000hello"), o); o=p.dumps("hello\u20ac"); // unicode string with non ascii Assert.AreEqual(B("X\u0008\u0000\u0000\u0000hello\u00e2\u0082\u00ac"), o); o=p.dumps((byte)'@'); Assert.AreEqual(B("K@"), o); o=p.dumps((sbyte)-40); Assert.AreEqual( B(new byte[]{(byte)'J',0xd8,0xff,0xff,0xff}), o); o=p.dumps((short)-0x1234); Assert.AreEqual(B("J\u00cc\u00ed\u00ff\u00ff"), o); o=p.dumps((ushort)0xf234); Assert.AreEqual(B("M\u0034\u00f2"), o); o=p.dumps((int)-0x12345678); Assert.AreEqual(B("J\u0088\u00a9\u00cb\u00ed"), o); o=p.dumps((uint)0x12345678); Assert.AreEqual(B(new byte[]{(byte)'J', 0x78, 0x56, 0x34, 0x12}), o); o=p.dumps((uint)0xf2345678); Assert.AreEqual(B("I4063516280\n"), o); o=p.dumps((long)0x12345678abcdefL); Assert.AreEqual(B("I5124095577148911\n"), o); o=p.dumps(1234.5678d); Assert.AreEqual(B(new byte[] {(byte)'G',0x40,0x93,0x4a,0x45,0x6d,0x5c,0xfa,0xad}), o); o=p.dumps(1234.5f); Assert.AreEqual(B(new byte[] {(byte)'G',0x40,0x93,0x4a,0,0,0,0,0}), o); o=p.dumps(1234.9876543210987654321m); Assert.AreEqual(B("cdecimal\nDecimal\nX\u0018\u0000\u0000\u00001234.9876543210987654321\u0085R"), o); DayEnum day=DayEnum.WEDNESDAY; o=p.dumps(day); // enum is returned as just a string representing the value Assert.AreEqual(B("X\u0009\u0000\u0000\u0000WEDNESDAY"),o); }
public void testMemoizationSet() { var set = new HashSet<string>(); set.Add("a"); object[] array = new object[] {set, set}; Pickler p = new Pickler(true); byte[] data = p.dumps(array); Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done Unpickler u = new Unpickler(); object[] result = (object[]) u.loads(data); Assert.AreEqual(2, result.Length); object first = result[0]; object second = result[1]; Assert.IsInstanceOf(typeof(HashSet<object>), first); Assert.IsInstanceOf(typeof(HashSet<object>), second); Assert.AreSame(first, second); // both objects should be the same memoized object HashSet<object> theSet = (HashSet<object>)second; Assert.AreEqual(1, theSet.Count); Assert.IsTrue(theSet.Contains("a")); }
public void testDates() { Pickler p=new Pickler(false); Unpickler u=new Unpickler(); DateTime date=new DateTime(2011,12,31,14,33,59); byte[] o = p.dumps(date); Object unpickled=u.loads(o); Assert.AreEqual(date, unpickled); date=new DateTime(2011,12,31,14,33,59,456); o=p.dumps(date); unpickled=u.loads(o); Assert.AreEqual(date, unpickled); }
public void TestRecursiveArray6() { Pickler p = new Pickler(false); object[] a = new object[] { "a","b","c","d","e","f" }; a[5] = a; // make it recursive p.dumps(a); }
public void TestRecursiveArray2() { Pickler p = new Pickler(false); object[] a = new object[] { "hello", "placeholder" }; a[1] = a; // make it recursive p.dumps(a); }
static void Main(string[] args) { PrintFiles(); int javaPort = int.Parse(Console.ReadLine()); Log("java_port: " + javaPort); var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); sock.Connect(IPAddress.Parse("127.0.0.1"), javaPort); using (NetworkStream s = new NetworkStream(sock)) { try { DateTime bootTime = DateTime.UtcNow; int splitIndex = SerDe.ReadInt(s); Log("split_index: " + splitIndex); if (splitIndex == -1) Environment.Exit(-1); string ver = SerDe.ReadString(s); Log("ver: " + ver); //// initialize global state //shuffle.MemoryBytesSpilled = 0 //shuffle.DiskBytesSpilled = 0 // fetch name of workdir string sparkFilesDir = SerDe.ReadString(s); Log("spark_files_dir: " + sparkFilesDir); //SparkFiles._root_directory = sparkFilesDir //SparkFiles._is_running_on_worker = True // fetch names of includes - not used //TODO - complete the impl int numberOfIncludesItems = SerDe.ReadInt(s); Log("num_includes: " + numberOfIncludesItems); if (numberOfIncludesItems > 0) { for (int i = 0; i < numberOfIncludesItems; i++) { string filename = SerDe.ReadString(s); } } // fetch names and values of broadcast variables int numBroadcastVariables = SerDe.ReadInt(s); Log("num_broadcast_variables: " + numBroadcastVariables); if (numBroadcastVariables > 0) { for (int i = 0; i < numBroadcastVariables; i++) { long bid = SerDe.ReadLong(s); if (bid >= 0) { string path = SerDe.ReadString(s); Broadcast.broadcastRegistry[bid] = new Broadcast(path); } else { bid = -bid - 1; Broadcast.broadcastRegistry.Remove(bid); } } } Accumulator.accumulatorRegistry.Clear(); int lengthOCommandByteArray = SerDe.ReadInt(s); Log("command_len: " + lengthOCommandByteArray); IFormatter formatter = new BinaryFormatter(); if (lengthOCommandByteArray > 0) { string deserializerMode = SerDe.ReadString(s); Log("Deserializer mode: " + deserializerMode); string serializerMode = SerDe.ReadString(s); Log("Serializer mode: " + serializerMode); byte[] command = SerDe.ReadBytes(s); Log("command bytes read: " + command.Length); var stream = new MemoryStream(command); var func = (Func<int, IEnumerable<dynamic>, IEnumerable<dynamic>>)formatter.Deserialize(stream); DateTime initTime = DateTime.UtcNow; int count = 0; foreach (var message in func(splitIndex, GetIterator(s, deserializerMode))) { byte[] buffer; if (serializerMode == "None") { buffer = message as byte[]; } else if (serializerMode == "String") { buffer = SerDe.ToBytes(message as string); } else if (serializerMode == "Row") { Pickler pickler = new Pickler(); buffer = pickler.dumps(new ArrayList { message }); } else { try { var ms = new MemoryStream(); formatter.Serialize(ms, message); buffer = ms.ToArray(); } catch (Exception) { Log(string.Format("{0} : {1}", message.GetType().Name, message.GetType().FullName)); throw; } } count++; SerDe.Write(s, buffer.Length); SerDe.Write(s, buffer); } //TODO - complete the impl Log("Count: " + count); //if profiler: // profiler.profile(process) //else: // process() DateTime finish_time = DateTime.UtcNow; SerDe.Write(s, (int)SpecialLengths.TIMING_DATA); SerDe.Write(s, ToUnixTime(bootTime)); SerDe.Write(s, ToUnixTime(initTime)); SerDe.Write(s, ToUnixTime(finish_time)); SerDe.Write(s, 0L); //shuffle.MemoryBytesSpilled SerDe.Write(s, 0L); //shuffle.DiskBytesSpilled } else { Log("Nothing to execute :-("); } // Mark the beginning of the accumulators section of the output SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION); SerDe.Write(s, Accumulator.accumulatorRegistry.Count); foreach (var item in Accumulator.accumulatorRegistry) { var ms = new MemoryStream(); var value = item.Value.GetType().GetField("value", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(item.Value); Log(string.Format("({0}, {1})", item.Key, value)); formatter.Serialize(ms, new KeyValuePair<int, dynamic>(item.Key, value)); byte[] buffer = ms.ToArray(); SerDe.Write(s, buffer.Length); SerDe.Write(s, buffer); } int end = SerDe.ReadInt(s); // check end of stream if (end == (int)SpecialLengths.END_OF_DATA_SECTION || end == (int)SpecialLengths.END_OF_STREAM) { SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM); Log("END_OF_STREAM: " + (int)SpecialLengths.END_OF_STREAM); } else { // write a different value to tell JVM to not reuse this worker SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION); Environment.Exit(-1); } s.Flush(); System.Threading.Thread.Sleep(1000); //TODO - not sure if this is really needed } catch (Exception e) { Log(e.ToString()); try { SerDe.Write(s, e.ToString()); } catch (IOException) { // JVM close the socket } catch (Exception ex) { LogError("CSharpWorker failed with exception:"); LogError(ex.ToString()); } Environment.Exit(-1); } } }
static void Main(string[] args) { // if there exists exe.config file, then use log4net if (File.Exists(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile)) { LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); } logger = LoggerServiceFactory.GetLogger(typeof(Worker)); Socket sock = null; try { PrintFiles(); int javaPort = int.Parse(Console.ReadLine()); logger.LogInfo("java_port: " + javaPort); sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); sock.Connect(IPAddress.Loopback, javaPort); } catch (Exception e) { logger.LogError("CSharpWorker failed with exception:"); logger.LogException(e); Environment.Exit(-1); } using (NetworkStream s = new NetworkStream(sock)) { try { DateTime bootTime = DateTime.UtcNow; int splitIndex = SerDe.ReadInt(s); logger.LogInfo("split_index: " + splitIndex); if (splitIndex == -1) Environment.Exit(-1); string ver = SerDe.ReadString(s); logger.LogInfo("ver: " + ver); //// initialize global state //shuffle.MemoryBytesSpilled = 0 //shuffle.DiskBytesSpilled = 0 // fetch name of workdir string sparkFilesDir = SerDe.ReadString(s); logger.LogInfo("spark_files_dir: " + sparkFilesDir); //SparkFiles._root_directory = sparkFilesDir //SparkFiles._is_running_on_worker = True // fetch names of includes - not used //TODO - complete the impl int numberOfIncludesItems = SerDe.ReadInt(s); logger.LogInfo("num_includes: " + numberOfIncludesItems); if (numberOfIncludesItems > 0) { for (int i = 0; i < numberOfIncludesItems; i++) { string filename = SerDe.ReadString(s); } } // fetch names and values of broadcast variables int numBroadcastVariables = SerDe.ReadInt(s); logger.LogInfo("num_broadcast_variables: " + numBroadcastVariables); if (numBroadcastVariables > 0) { for (int i = 0; i < numBroadcastVariables; i++) { long bid = SerDe.ReadLong(s); if (bid >= 0) { string path = SerDe.ReadString(s); Broadcast.broadcastRegistry[bid] = new Broadcast(path); } else { bid = -bid - 1; Broadcast.broadcastRegistry.Remove(bid); } } } Accumulator.accumulatorRegistry.Clear(); int lengthOCommandByteArray = SerDe.ReadInt(s); logger.LogInfo("command_len: " + lengthOCommandByteArray); IFormatter formatter = new BinaryFormatter(); if (lengthOCommandByteArray > 0) { Stopwatch commandProcessWatch = new Stopwatch(); Stopwatch funcProcessWatch = new Stopwatch(); commandProcessWatch.Start(); int rddId = SerDe.ReadInt(s); int stageId = SerDe.ReadInt(s); int partitionId = SerDe.ReadInt(s); logger.LogInfo(string.Format("rddInfo: rddId {0}, stageId {1}, partitionId {2}", rddId, stageId, partitionId)); string deserializerMode = SerDe.ReadString(s); logger.LogInfo("Deserializer mode: " + deserializerMode); string serializerMode = SerDe.ReadString(s); logger.LogInfo("Serializer mode: " + serializerMode); byte[] command = SerDe.ReadBytes(s); logger.LogInfo("command bytes read: " + command.Length); var stream = new MemoryStream(command); var workerFunc = (CSharpWorkerFunc)formatter.Deserialize(stream); var func = workerFunc.Func; logger.LogInfo(string.Format("stack trace of workerFunc (dont't panic, this is just for debug):\n{0}", workerFunc.StackTrace)); DateTime initTime = DateTime.UtcNow; int count = 0; // here we use low level API because we need to get perf metrics WorkerInputEnumerator inputEnumerator = new WorkerInputEnumerator(s, deserializerMode); IEnumerable<dynamic> inputEnumerable = Enumerable.Cast<dynamic>(inputEnumerator); funcProcessWatch.Start(); IEnumerable<dynamic> outputEnumerable = func(splitIndex, inputEnumerable); var outputEnumerator = outputEnumerable.GetEnumerator(); funcProcessWatch.Stop(); while (true) { funcProcessWatch.Start(); bool hasNext = outputEnumerator.MoveNext(); funcProcessWatch.Stop(); if (!hasNext) { break; } funcProcessWatch.Start(); var message = outputEnumerator.Current; funcProcessWatch.Stop(); if (object.ReferenceEquals(null, message)) { continue; } byte[] buffer; if (serializerMode == "None") { buffer = message as byte[]; } else if (serializerMode == "String") { buffer = SerDe.ToBytes(message as string); } else if (serializerMode == "Row") { Pickler pickler = new Pickler(); buffer = pickler.dumps(new ArrayList { message }); } else { try { var ms = new MemoryStream(); formatter.Serialize(ms, message); buffer = ms.ToArray(); } catch (Exception) { logger.LogError(string.Format("{0} : {1}", message.GetType().Name, message.GetType().FullName)); throw; } } count++; SerDe.Write(s, buffer.Length); SerDe.Write(s, buffer); } //TODO - complete the impl logger.LogInfo("Count: " + count); //if profiler: // profiler.profile(process) //else: // process() DateTime finish_time = DateTime.UtcNow; const string format = "MM/dd/yyyy hh:mm:ss.fff tt"; logger.LogInfo(string.Format("bootTime: {0}, initTime: {1}, finish_time: {2}", bootTime.ToString(format), initTime.ToString(format), finish_time.ToString(format))); SerDe.Write(s, (int)SpecialLengths.TIMING_DATA); SerDe.Write(s, ToUnixTime(bootTime)); SerDe.Write(s, ToUnixTime(initTime)); SerDe.Write(s, ToUnixTime(finish_time)); SerDe.Write(s, 0L); //shuffle.MemoryBytesSpilled SerDe.Write(s, 0L); //shuffle.DiskBytesSpilled commandProcessWatch.Stop(); // log statistics inputEnumerator.LogStatistic(); logger.LogInfo(string.Format("func process time: {0}", funcProcessWatch.ElapsedMilliseconds)); logger.LogInfo(string.Format("command process time: {0}", commandProcessWatch.ElapsedMilliseconds)); } else { logger.LogWarn("Nothing to execute :-("); } // Mark the beginning of the accumulators section of the output SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION); SerDe.Write(s, Accumulator.accumulatorRegistry.Count); foreach (var item in Accumulator.accumulatorRegistry) { var ms = new MemoryStream(); var value = item.Value.GetType().GetField("value", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(item.Value); logger.LogInfo(string.Format("({0}, {1})", item.Key, value)); formatter.Serialize(ms, new KeyValuePair<int, dynamic>(item.Key, value)); byte[] buffer = ms.ToArray(); SerDe.Write(s, buffer.Length); SerDe.Write(s, buffer); } int end = SerDe.ReadInt(s); // check end of stream if (end == (int)SpecialLengths.END_OF_STREAM) { SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM); logger.LogInfo("END_OF_STREAM: " + (int)SpecialLengths.END_OF_STREAM); } else { // write a different value to tell JVM to not reuse this worker SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION); Environment.Exit(-1); } s.Flush(); // log bytes read and write logger.LogInfo(string.Format("total read bytes: {0}", SerDe.totalReadNum)); logger.LogInfo(string.Format("total write bytes: {0}", SerDe.totalWriteNum)); // wait for server to complete, otherwise server gets 'connection reset' exception // Use SerDe.ReadBytes() to detect java side has closed socket properly // ReadBytes() will block until the socket is closed SerDe.ReadBytes(s); } catch (Exception e) { logger.LogError(e.ToString()); try { SerDe.Write(s, e.ToString()); } catch (IOException) { // JVM close the socket } catch (Exception ex) { logger.LogError("CSharpWorker failed with exception:"); logger.LogException(ex); } Environment.Exit(-1); } } sock.Close(); }
public void testFailure() { NotABean notabean=new NotABean(); notabean.x=42; Pickler p=new Pickler(false); p.dumps(notabean); }
public void testDateTimeStringEscaping() { DateTime dt=new DateTime(2011, 10, 10, 9, 13, 10, 10); Pickler p = new Pickler(); byte[] pickle = p.dumps(dt); Unpickler u = new Unpickler(); DateTime dt2 = (DateTime) u.loads(pickle); Assert.AreEqual(dt, dt2); dt = new DateTime(2011, 10, 9, 13, 10, 9, 10); dt2 = (DateTime) U("\u0080\u0002cdatetime\ndatetime\nq\u0000U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001\u0085q\u0002Rq\u0003."); // protocol 2 Assert.AreEqual(dt, dt2); dt2 = (DateTime) U("cdatetime\ndatetime\nq\u0000(U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001tq\u0002Rq\u0003."); // protocol 1 Assert.AreEqual(dt, dt2); dt2 = (DateTime) U("cdatetime\ndatetime\np0\n(S'\\x07\\xdb\\n\\t\\r\\n\\t\\x00'\\x10'\np1\ntp2\nRp3\n."); // protocol 0 Assert.AreEqual(dt, dt2); }
public void testCustomPickler() { Pickler.registerCustomPickler(typeof(CustomClass), new CustomClassPickler()); CustomClass c=new CustomClass(); Pickler p=new Pickler(false); byte[] ser = p.dumps(c); Unpickler u = new Unpickler(); string x = (string) u.loads(ser); Assert.AreEqual("customclassint=42", x); }
public void testZeroToTwoFiveSix() { byte[] bytes=new byte[256]; for(int b=0; b<256; ++b) { bytes[b]=(byte)b; } StringBuilder sb=new StringBuilder(); for(int i=0; i<256; ++i) { sb.Append((char)i); } string str=sb.ToString(); Pickler p=new Pickler(false); Unpickler u=new Unpickler(); MemoryStream bos=new MemoryStream(434); bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2); byte[] data=Encoding.Default.GetBytes("c__builtin__\nbytearray\n"); bos.Write(data,0,data.Length); bos.WriteByte(Opcodes.BINUNICODE); bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4); byte[] utf8=Encoding.UTF8.GetBytes(str); bos.Write(utf8,0,utf8.Length); bos.WriteByte(Opcodes.BINUNICODE); bos.Write(new byte[] {7,0,0,0},0,4); data=Encoding.Default.GetBytes("latin-1"); bos.Write(data,0,data.Length); bos.WriteByte(Opcodes.TUPLE2); bos.WriteByte(Opcodes.REDUCE); bos.WriteByte(Opcodes.STOP); byte[] bytesresult=bos.ToArray(); byte[] output=p.dumps(bytes); Assert.AreEqual(bytesresult, output); Assert.AreEqual(bytes, (byte[])u.loads(output)); bos=new MemoryStream(434); bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2); bos.WriteByte(Opcodes.BINUNICODE); bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4); utf8=Encoding.UTF8.GetBytes(str); bos.Write(utf8,0,utf8.Length); bos.WriteByte(Opcodes.STOP); bytesresult=bos.ToArray(); output=p.dumps(str); Assert.AreEqual(bytesresult, output); Assert.AreEqual(str, u.loads(output)); }
public void testTimes() { Pickler p=new Pickler(false); Unpickler u=new Unpickler(); TimeSpan ts=new TimeSpan(2, 0, 0, 7000, 456); byte[] o = p.dumps(ts); object unpickled=u.loads(o); Assert.AreEqual(ts,unpickled); Assert.AreEqual(B("cdatetime\ntimedelta\nK\u0002MX\u001bJ@\u00f5\u0006\u0000\u0087R"), o); }
public void testMemoizationString() { string str = "a"; object[] array = new object[] {str, str}; Pickler p = new Pickler(true); byte[] data = p.dumps(array); Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done Unpickler u = new Unpickler(); object[] result = (object[]) u.loads(data); Assert.AreEqual(2, result.Length); object first = result[0]; object second = result[1]; Assert.IsInstanceOf(typeof(string), first); Assert.IsInstanceOf(typeof(string), second); Assert.AreSame(first, second); // both objects should be the same memoized object str = (string) second; Assert.AreEqual("a", str); }
private static byte[] GetSerializedMessage(string serializerMode, dynamic message, IFormatter formatter) { byte[] buffer; switch ((SerializedMode)Enum.Parse(typeof(SerializedMode), serializerMode)) { case SerializedMode.None: buffer = message as byte[]; break; case SerializedMode.String: buffer = SerDe.ToBytes(message as string); break; case SerializedMode.Row: var pickler = new Pickler(); buffer = pickler.dumps(new ArrayList { message }); break; default: try { var ms = new MemoryStream(); formatter.Serialize(ms, message); buffer = ms.ToArray(); } catch (Exception) { logger.LogError("Exception serializing output"); logger.LogError("{0} : {1}", message.GetType().Name, message.GetType().FullName); throw; } break; } return buffer; }
public void testSets() { byte[] o; Pickler p=new Pickler(false); Unpickler up=new Unpickler(); var intset=new HashSet<int>(); intset.Add(1); intset.Add(2); intset.Add(3); o=p.dumps(intset); HashSet<object> resultset=(HashSet<object>)up.loads(o); AssertUtils.AssertEqual(intset, resultset); HashSet<string> stringset=new HashSet<string>(); stringset.Add("A"); stringset.Add("B"); stringset.Add("C"); o=p.dumps(stringset); resultset=(HashSet<object>)up.loads(o); AssertUtils.AssertEqual(stringset, resultset); }
public void testUnpicklingPerformance() { Pickler pickler = new Pickler(); var myList = new List<string>(); for (int i = 0; i < 10; i++) { myList.Add(i.ToString()); } byte[] bytes = pickler.dumps(myList); Unpickler unpickler = new Unpickler(); DateTime start = DateTime.Now; for (int i = 0; i < 1000000; i++) { unpickler.loads(bytes); } Console.WriteLine(DateTime.Now - start); }
public void testMappings() { byte[] o; Pickler p=new Pickler(false); Unpickler pu=new Unpickler(); var intmap=new Dictionary<int,int>(); intmap.Add(1, 11); intmap.Add(2, 22); intmap.Add(3, 33); o=p.dumps(intmap); Hashtable resultmap=(Hashtable)pu.loads(o); AssertUtils.AssertEqual(intmap, resultmap); var stringmap=new Dictionary<string,string>(); stringmap.Add("A", "1"); stringmap.Add("B", "2"); stringmap.Add("C", "3"); o=p.dumps(stringmap); resultmap=(Hashtable)pu.loads(o); AssertUtils.AssertEqual(stringmap, resultmap); Hashtable table=new Hashtable(); table.Add(1,11); table.Add(2,22); table.Add(3,33); o=p.dumps(table); resultmap=(Hashtable)pu.loads(o); AssertUtils.AssertEqual(table, resultmap); }
public void TestWorkerWithRowDeserializedModeAndBytesSerializedMode() { Process worker; var CSharpRDD_SocketServer = CreateServer(out worker); const int expectedCount = 5; using (var serverSocket = CSharpRDD_SocketServer.Accept()) using (var s = serverSocket.GetStream()) { WritePayloadHeaderToWorker(s); byte[] commandWithRowDeserializeMode = SparkContext.BuildCommand(new CSharpWorkerFunc((pid, iter) => iter), SerializedMode.Row); SerDe.Write(s, commandWithRowDeserializeMode.Length); SerDe.Write(s, commandWithRowDeserializeMode); new StructTypePickler().Register(); new RowPickler().Register(); Pickler pickler = new Pickler(); for (int i = 0; i < expectedCount; i++) { byte[] pickleBytes = pickler.dumps(new[] { RowHelper.BuildRowForBasicSchema(i) }); SerDe.Write(s, pickleBytes.Length); SerDe.Write(s, pickleBytes); } SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION); SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM); s.Flush(); int count = 0; var formatter = new BinaryFormatter(); foreach (var bytes in ReadWorker(s)) { var ms = new MemoryStream(bytes); var rows = new ArrayList { formatter.Deserialize(ms) }.Cast<Row>().ToArray(); Assert.AreEqual(1, rows.Count()); Assert.AreEqual(count, rows[0].Get("age")); count++; } Assert.AreEqual(expectedCount, count); } AssertWorker(worker); CSharpRDD_SocketServer.Close(); }
public void testLists() { byte[] o; Pickler p=new Pickler(false); IList list=new ArrayList(); list.Add(1); list.Add("abc"); list.Add(null); o=p.dumps(list); Assert.AreEqual(B("](K\u0001X\u0003\u0000\u0000\u0000abcNe"), o); IList<object> ilist=new List<object>(); ilist.Add(1); ilist.Add("abc"); ilist.Add(null); o=p.dumps(ilist); Assert.AreEqual(B("](K\u0001X\u0003\u0000\u0000\u0000abcNe"), o); Stack<int> stack=new Stack<int>(); stack.Push(1); stack.Push(2); stack.Push(3); o=p.dumps(stack); Assert.AreEqual(B("](K\u0003K\u0002K\u0001e"), o); var queue=new Queue<int>(); queue.Enqueue(1); queue.Enqueue(2); queue.Enqueue(3); o=p.dumps(queue); Assert.AreEqual(B("](K\u0001K\u0002K\u0003e"), o); }