object U(byte[] data) { Unpickler u=new Unpickler(); object o=u.loads(data); u.close(); return o; }
public override object deserializeData(byte[] data) { using(var u=new Unpickler()) { return u.loads(data); } }
/// <summary> /// Unpickles objects from byte[] /// </summary> /// <param name="buffer"></param> /// <returns></returns> internal static object[] GetUnpickledObjects(byte[] buffer) { var unpickler = new Unpickler(); //not making any assumptions about the implementation and hence not a class member var unpickledItems = unpickler.loads(buffer); Debug.Assert(unpickledItems != null); return (unpickledItems as object[]); }
object U(byte[] data) { using(Unpickler u=new Unpickler()) { return u.loads(data); } }
public static void Main(string[] args) { Unpickler u=new Unpickler(); byte[] data; object result; Console.WriteLine("here we go; 1"); data=PickleUtils.str2bytes("\u0080\u0002carray\narray\nq\u0000U\u0001iq\u0001]q\u0002\u0086q\u0003Rq\u0004."); result=u.loads(data); PrettyPrint.print(result); Console.WriteLine("here we go; 2"); data=PickleUtils.str2bytes("\u0080\u0003carray\n_array_reconstructor\nq\u0000(carray\narray\nq\u0001X\u0001\u0000\u0000\u0000iq\u0002K\u0008C\u000c\u000f'\u0000\u0000\u00b8\"\u0000\u0000a\u001e\u0000\u0000q\u0003tq\u0004Rq\u0005."); result=u.loads(data); PrettyPrint.print(result); Console.Write("Press any key to continue . . . "); Console.ReadKey(true); }
public void testAnonType() { var x = new { Name="Harry", Country="UK", Age=34 }; Pickler p = new Pickler(); byte[] ser = p.dumps(x); Unpickler u = new Unpickler(); object y = u.loads(ser); IDictionary dict = (IDictionary) y; Assert.AreEqual(3, dict.Count); Assert.AreEqual(34, dict["Age"]); Assert.IsFalse(dict.Contains("__class__")); }
public static void Main(string[] args) { // going to pickle a c# datastructure var map = new Dictionary<string, object>(); map["apple"] = 42; map["microsoft"] = "hello"; var values = new List<double>(); values.AddRange(new double[] { 1.11, 2.22, 3.33, 4.44, 5.55} ); map["values"] = values; // You can add many other types if you like. See the readme about the type mappings. const string PickleFilename = "testpickle.dat"; Console.WriteLine("Writing pickle to '{0}'", PickleFilename); var pickler = new Pickler(true); using(FileStream fos = new FileStream(PickleFilename, FileMode.Create)) { pickler.dump(map, fos); } Console.WriteLine("Done. Try unpickling it in python.\n"); Console.WriteLine("Reading a pickle created in python..."); // the following pickle was created in Python 3.4. // it is this data: [1, 2, 3, (11, 12, 13), {'banana', 'grape', 'apple'}] byte[] pythonpickle = new byte[] {128, 4, 149, 48, 0, 0, 0, 0, 0, 0, 0, 93, 148, 40, 75, 1, 75, 2, 75, 3, 75, 11, 75, 12, 75, 13, 135, 148, 143, 148, 40, 140, 6, 98, 97, 110, 97, 110, 97, 148, 140, 5, 103, 114, 97, 112, 101, 148, 140, 5, 97, 112, 112, 108, 101, 148, 144, 101, 46}; var unpickler = new Unpickler(); object result = unpickler.loads(pythonpickle); Console.WriteLine("type: {0}", result.GetType()); var list = (ArrayList) result; int integer1 = (int)list[0]; int integer2 = (int)list[1]; int integer3 = (int)list[2]; object[] tuple = (object[]) list[3]; HashSet<object> set = (HashSet<object>) list[4]; Console.WriteLine("1-3: integers: {0}, {1}, {2}", integer1, integer2, integer3); Console.WriteLine("4: tuple: ({0}, {1}, {2})", tuple[0], tuple[1], tuple[2]); Console.WriteLine("5: set: {0}", string.Join(",", set)); Console.WriteLine("\r\nEnter to exit:"); Console.ReadLine(); }
private static IEnumerable<dynamic> GetIterator(ISparkCLRSocket socket, string serializedMode) { Log("Serialized mode in GetIterator: " + serializedMode); IFormatter formatter = new BinaryFormatter(); int messageLength; do { messageLength = socket.ReadInt(); if (messageLength > 0) { byte[] buffer = socket.ReadBytes(messageLength); switch (serializedMode) { case "String": yield return SerDe.ToString(buffer); break; case "Row": Unpickler unpickler = new Unpickler(); foreach (var item in (unpickler.loads(buffer) as object[])) { yield return item; } break; case "Pair": messageLength = socket.ReadInt(); byte[] value = socket.ReadBytes(messageLength); yield return new KeyValuePair<byte[], byte[]>(buffer, value); break; case "Byte": default: var ms = new MemoryStream(buffer); dynamic message = formatter.Deserialize(ms); yield return message; break; } } } while (messageLength >= 0); }
private static IEnumerable<dynamic> GetIterator(Stream s, string serializedMode) { Log("Serialized mode in GetIterator: " + serializedMode); IFormatter formatter = new BinaryFormatter(); int messageLength; while ((messageLength = SerDe.ReadInt(s)) != (int)SpecialLengths.END_OF_DATA_SECTION) { if (messageLength > 0 || serializedMode == "Pair") { byte[] buffer = messageLength > 0 ? SerDe.ReadBytes(s, messageLength) : null; switch (serializedMode) { case "String": yield return SerDe.ToString(buffer); break; case "Row": Unpickler unpickler = new Unpickler(); foreach (var item in (unpickler.loads(buffer) as object[])) { yield return item; } break; case "Pair": messageLength = SerDe.ReadInt(s); if (messageLength > 0) { yield return new KeyValuePair<byte[], byte[]>(buffer, SerDe.ReadBytes(s, messageLength)); } break; case "Byte": default: var ms = new MemoryStream(buffer); dynamic message = formatter.Deserialize(ms); yield return message; break; } } } }
public void testMemoizationCollection() { ICollection<string> list = new List<string>(); list.Add("a"); object[] array = new object[] {list, list}; Pickler p = new Pickler(true); byte[] data = p.dumps(array); Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done Unpickler u = new Unpickler(); object[] result = (object[]) u.loads(data); Assert.AreEqual(2, result.Length); object first = result[0]; object second = result[1]; Assert.IsInstanceOf(typeof(ArrayList), first); Assert.IsInstanceOf(typeof(ArrayList), second); Assert.AreSame(first, second); // both objects should be the same memoized object ArrayList theList = (ArrayList) second; Assert.AreEqual(1, theList.Count); Assert.IsTrue(theList.Contains("a")); }
public void testCustomPickler() { Pickler.registerCustomPickler(typeof(CustomClass), new CustomClassPickler()); CustomClass c=new CustomClass(); Pickler p=new Pickler(false); byte[] ser = p.dumps(c); Unpickler u = new Unpickler(); string x = (string) u.loads(ser); Assert.AreEqual("customclassint=42", x); }
public void testClass() { Pickler p=new Pickler(false); Unpickler pu=new Unpickler(); byte[] o; Relative person=new Relative("Tupac",true, new int[] {3,4,5}); o=p.dumps(person); Hashtable map=(Hashtable) pu.loads(o); Assert.AreEqual(5, map.Count); Assert.AreEqual("Pyrolite.Tests.Pickle.PicklerTests+Relative", map["__class__"]); Assert.AreEqual("Tupac", map["Name"]); Assert.AreEqual("unspecified", map["Relation"]); Assert.AreEqual(true, map["Deceased"]); Assert.AreEqual(new int[] {3,4,5}, map["Values"]); }
public void testMemoizationRecursiveMemo() { byte[] o; Pickler p=new Pickler(); // self-referencing list string reused = "reused"; IList list=new ArrayList(); list.Add(reused); list.Add(reused); list.Add(list); o=p.dumps(list); Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01h\x0000e.", S(o)); Unpickler u = new Unpickler(); ArrayList data = (ArrayList) u.loads(o); Assert.AreEqual(3, data.Count); string s1 = (string) data[0]; string s2 = (string) data[1]; ArrayList data2 = (ArrayList) data[2]; Assert.AreEqual("reused", s1); Assert.AreSame(s1, s2); Assert.AreSame(data, data2); Assert.AreSame(data[0], data2[0]); // self-referencing hashtable Hashtable h = new Hashtable(); h["myself"] = h; o=p.dumps(h); Assert.AreEqual("\x80\x02}q\x00(X\x06\x00\x00\x0000myselfq\x01h\x0000u.", S(o)); Hashtable h2 = (Hashtable) u.loads(o); Assert.AreEqual(1, h2.Count); Assert.AreSame(h2, h2["myself"]); }
public void testMemoization() { byte[] o; Pickler p=new Pickler(); string reused = "reused"; string another = "another"; IList list=new ArrayList(); IList sublist = new ArrayList(); sublist.Add(reused); sublist.Add(reused); sublist.Add(another); list.Add(reused); list.Add(reused); list.Add(another); list.Add(sublist); o=p.dumps(list); Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01X\x07\x00\x00\x0000anotherq\x02]q\x03(h\x01h\x01h\x0002ee.", S(o)); Unpickler u = new Unpickler(); ArrayList data = (ArrayList) u.loads(o); Assert.AreEqual(4, data.Count); string s1 = (string) data[0]; string s2 = (string) data[1]; string s3 = (string) data[2]; data = (ArrayList) data[3]; string s4 = (string) data[0]; string s5 = (string) data[1]; string s6 = (string) data[2]; Assert.AreEqual("reused", s1); Assert.AreEqual("another", s3); Assert.AreSame(s1, s2); Assert.AreSame(s3, s6); Assert.AreSame(s1, s4); Assert.AreSame(s1, s5); }
public void testMappings() { byte[] o; Pickler p=new Pickler(false); Unpickler pu=new Unpickler(); var intmap=new Dictionary<int,int>(); intmap.Add(1, 11); intmap.Add(2, 22); intmap.Add(3, 33); o=p.dumps(intmap); Hashtable resultmap=(Hashtable)pu.loads(o); AssertUtils.AssertEqual(intmap, resultmap); var stringmap=new Dictionary<string,string>(); stringmap.Add("A", "1"); stringmap.Add("B", "2"); stringmap.Add("C", "3"); o=p.dumps(stringmap); resultmap=(Hashtable)pu.loads(o); AssertUtils.AssertEqual(stringmap, resultmap); Hashtable table=new Hashtable(); table.Add(1,11); table.Add(2,22); table.Add(3,33); o=p.dumps(table); resultmap=(Hashtable)pu.loads(o); AssertUtils.AssertEqual(table, resultmap); }
public void testMemoizationString() { string str = "a"; object[] array = new object[] {str, str}; Pickler p = new Pickler(true); byte[] data = p.dumps(array); Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done Unpickler u = new Unpickler(); object[] result = (object[]) u.loads(data); Assert.AreEqual(2, result.Length); object first = result[0]; object second = result[1]; Assert.IsInstanceOf(typeof(string), first); Assert.IsInstanceOf(typeof(string), second); Assert.AreSame(first, second); // both objects should be the same memoized object str = (string) second; Assert.AreEqual("a", str); }
public void testZeroToTwoFiveSix() { byte[] bytes=new byte[256]; for(int b=0; b<256; ++b) { bytes[b]=(byte)b; } StringBuilder sb=new StringBuilder(); for(int i=0; i<256; ++i) { sb.Append((char)i); } string str=sb.ToString(); Pickler p=new Pickler(false); Unpickler u=new Unpickler(); MemoryStream bos=new MemoryStream(434); bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2); byte[] data=Encoding.Default.GetBytes("c__builtin__\nbytearray\n"); bos.Write(data,0,data.Length); bos.WriteByte(Opcodes.BINUNICODE); bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4); byte[] utf8=Encoding.UTF8.GetBytes(str); bos.Write(utf8,0,utf8.Length); bos.WriteByte(Opcodes.BINUNICODE); bos.Write(new byte[] {7,0,0,0},0,4); data=Encoding.Default.GetBytes("latin-1"); bos.Write(data,0,data.Length); bos.WriteByte(Opcodes.TUPLE2); bos.WriteByte(Opcodes.REDUCE); bos.WriteByte(Opcodes.STOP); byte[] bytesresult=bos.ToArray(); byte[] output=p.dumps(bytes); Assert.AreEqual(bytesresult, output); Assert.AreEqual(bytes, (byte[])u.loads(output)); bos=new MemoryStream(434); bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2); bos.WriteByte(Opcodes.BINUNICODE); bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4); utf8=Encoding.UTF8.GetBytes(str); bos.Write(utf8,0,utf8.Length); bos.WriteByte(Opcodes.STOP); bytesresult=bos.ToArray(); output=p.dumps(str); Assert.AreEqual(bytesresult, output); Assert.AreEqual(str, u.loads(output)); }
public void testDateTimeStringEscaping() { DateTime dt=new DateTime(2011, 10, 10, 9, 13, 10, 10); Pickler p = new Pickler(); byte[] pickle = p.dumps(dt); Unpickler u = new Unpickler(); DateTime dt2 = (DateTime) u.loads(pickle); Assert.AreEqual(dt, dt2); dt = new DateTime(2011, 10, 9, 13, 10, 9, 10); dt2 = (DateTime) U("\u0080\u0002cdatetime\ndatetime\nq\u0000U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001\u0085q\u0002Rq\u0003."); // protocol 2 Assert.AreEqual(dt, dt2); dt2 = (DateTime) U("cdatetime\ndatetime\nq\u0000(U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001tq\u0002Rq\u0003."); // protocol 1 Assert.AreEqual(dt, dt2); dt2 = (DateTime) U("cdatetime\ndatetime\np0\n(S'\\x07\\xdb\\n\\t\\r\\n\\t\\x00'\\x10'\np1\ntp2\nRp3\n."); // protocol 0 Assert.AreEqual(dt, dt2); }
public void testMemoizationMap() { var map = new Dictionary<string,string>(); map.Add("key", "value"); object[] array = new object[] {map, map}; Pickler p = new Pickler(true); byte[] data = p.dumps(array); Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done Unpickler u = new Unpickler(); object[] result = (object[]) u.loads(data); Assert.AreEqual(2, result.Length); object first = result[0]; object second = result[1]; Assert.IsInstanceOf(typeof(Hashtable), first); Assert.IsInstanceOf(typeof(Hashtable), second); Assert.AreSame(first, second); // both objects should be the same memoized object Hashtable theMap = (Hashtable) second; Assert.AreEqual(1, theMap.Count); Assert.AreEqual("value", theMap["key"]); }
public void testMemoizationSet() { var set = new HashSet<string>(); set.Add("a"); object[] array = new object[] {set, set}; Pickler p = new Pickler(true); byte[] data = p.dumps(array); Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done Unpickler u = new Unpickler(); object[] result = (object[]) u.loads(data); Assert.AreEqual(2, result.Length); object first = result[0]; object second = result[1]; Assert.IsInstanceOf(typeof(HashSet<object>), first); Assert.IsInstanceOf(typeof(HashSet<object>), second); Assert.AreSame(first, second); // both objects should be the same memoized object HashSet<object> theSet = (HashSet<object>)second; Assert.AreEqual(1, theSet.Count); Assert.IsTrue(theSet.Contains("a")); }
/// <summary> /// Unpickles objects from byte[] /// </summary> /// <param name="buffer"></param> /// <returns></returns> internal static object[] GetUnpickledObjects(byte[] buffer) { var unpickler = new Unpickler(); //not making any assumptions about the implementation and hence not a class member return (unpickler.loads(buffer) as object[]); }
public void testBinint2WithObject() { Unpickler u=new Unpickler(); Unpickler.registerConstructor("Pyro4.core", "URI", new StringConstructor()); byte[] data=PickleUtils.str2bytes("\u0080\u0002cPyro4.core\nURI\n)\u0081M\u0082#."); int result=(int) u.loads(data); Assert.AreEqual(9090,result); }
public void testMemoizationDecimal() { decimal bigd = 12345678901234567890.99887766m; object[] array = new object[] {bigd, bigd}; Pickler p = new Pickler(true); byte[] data = p.dumps(array); Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done Unpickler u = new Unpickler(); object[] result = (object[]) u.loads(data); Assert.AreEqual(2, result.Length); Assert.IsInstanceOf(typeof(decimal), result[0]); Assert.IsInstanceOf(typeof(decimal), result[1]); Assert.AreSame(result[0], result[1]); // both objects should be the same memoized object bigd = (decimal) result[1]; Assert.AreEqual(12345678901234567890.99887766m, bigd); }
public void testMemoizationTimeStuff() { TimeSpan delta = new TimeSpan(1,2,3); DateTime time = new DateTime(2014,11,20,1,2,3); object[] array = new object[] {delta, delta, time, time}; Pickler p = new Pickler(true); byte[] data = p.dumps(array); Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done Unpickler u = new Unpickler(); object[] result = (object[]) u.loads(data); Assert.AreEqual(4, result.Length); Assert.IsInstanceOf(typeof(TimeSpan), result[0]); Assert.IsInstanceOf(typeof(TimeSpan), result[1]); Assert.IsInstanceOf(typeof(DateTime), result[2]); Assert.IsInstanceOf(typeof(DateTime), result[3]); Assert.AreSame(result[0], result[1]); // both objects should be the same memoized object Assert.AreSame(result[2], result[3]); // both objects should be the same memoized object delta = (TimeSpan) result[1]; time = (DateTime) result[3]; Assert.AreEqual(new TimeSpan(1,2,3), delta); Assert.AreEqual(new DateTime(2014,11,20,1,2,3), time); }
public void testBinint2WithObject() { Unpickler u=new Unpickler(); byte[] data=PickleUtils.str2bytes("\u0080\u0002cIgnore.Ignore\nIgnore\n)\u0081M\u0082#."); int result=(int) u.loads(data); Assert.AreEqual(9090,result); }
public void testDates() { Pickler p=new Pickler(false); Unpickler u=new Unpickler(); DateTime date=new DateTime(2011,12,31,14,33,59); byte[] o = p.dumps(date); Object unpickled=u.loads(o); Assert.AreEqual(date, unpickled); date=new DateTime(2011,12,31,14,33,59,456); o=p.dumps(date); unpickled=u.loads(o); Assert.AreEqual(date, unpickled); }
public void testUnpicklingPerformance() { Pickler pickler = new Pickler(); var myList = new List<string>(); for (int i = 0; i < 10; i++) { myList.Add(i.ToString()); } byte[] bytes = pickler.dumps(myList); Unpickler unpickler = new Unpickler(); DateTime start = DateTime.Now; for (int i = 0; i < 1000000; i++) { unpickler.loads(bytes); } Console.WriteLine(DateTime.Now - start); }
public void testTimes() { Pickler p=new Pickler(false); Unpickler u=new Unpickler(); TimeSpan ts=new TimeSpan(2, 0, 0, 7000, 456); byte[] o = p.dumps(ts); object unpickled=u.loads(o); Assert.AreEqual(ts,unpickled); Assert.AreEqual(B("cdatetime\ntimedelta\nK\u0002MX\u001bJ@\u00f5\u0006\u0000\u0087R"), o); }
public void TestWorkerWithBytesDeserializedModeAndRowSerializedMode() { const int expectedCount = 100; Process worker; var CSharpRDD_SocketServer = CreateServer(out worker); using (var serverSocket = CSharpRDD_SocketServer.Accept()) using (var s = serverSocket.GetStream()) { WritePayloadHeaderToWorker(s); byte[] command = SparkContext.BuildCommand(new CSharpWorkerFunc((pid, iter) => iter), SerializedMode.Byte, SerializedMode.Row); SerDe.Write(s, command.Length); SerDe.Write(s, command); var formatter = new BinaryFormatter(); for (int i = 0; i < expectedCount; i++) { var ms = new MemoryStream(); formatter.Serialize(ms, i); var buffer = ms.ToArray(); SerDe.Write(s, buffer.Length); SerDe.Write(s, ms.ToArray()); } SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION); SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM); s.Flush(); lock (syncLock) { Console.WriteLine(output); } int count = 0; Unpickler unpickler = new Unpickler(); foreach (var bytes in ReadWorker(s)) { var objects = unpickler.loads(bytes) as ArrayList; Assert.IsNotNull(objects); Assert.IsTrue(objects.Count == 1); Assert.AreEqual(count++, (int)objects[0]); } Assert.AreEqual(expectedCount, count); } AssertWorker(worker); CSharpRDD_SocketServer.Close(); }
public void testSets() { byte[] o; Pickler p=new Pickler(false); Unpickler up=new Unpickler(); var intset=new HashSet<int>(); intset.Add(1); intset.Add(2); intset.Add(3); o=p.dumps(intset); HashSet<object> resultset=(HashSet<object>)up.loads(o); AssertUtils.AssertEqual(intset, resultset); HashSet<string> stringset=new HashSet<string>(); stringset.Add("A"); stringset.Add("B"); stringset.Add("C"); o=p.dumps(stringset); resultset=(HashSet<object>)up.loads(o); AssertUtils.AssertEqual(stringset, resultset); }