public UnpicklerImplementation(T input, IDictionary <int, object> memo, UnpickleStack stack, Unpickler unpickler)
{
this.input = input;
this.memo = memo;
this.stack = stack;
this.unpickler = unpickler;
}
public override object deserializeData(byte[] data)
{
using(var u=new Unpickler())
{
return u.loads(data);
}
}
object U(byte[] data)
{
Unpickler u=new Unpickler();
object o=u.loads(data);
u.close();
return o;
}
/// <summary>
/// Unpickles objects from byte[]
/// </summary>
/// <param name="buffer"></param>
/// <returns></returns>
internal static object[] GetUnpickledObjects(byte[] buffer)
{
var unpickler = new Unpickler(); //not making any assumptions about the implementation and hence not a class member
var unpickledItems = unpickler.loads(buffer);
Debug.Assert(unpickledItems != null);
return (unpickledItems as object[]);
}
object U(byte[] data)
{
using(Unpickler u=new Unpickler())
{
return u.loads(data);
}
}
public void testAnonType()
{
var x = new { Name="Harry", Country="UK", Age=34 };
Pickler p = new Pickler();
byte[] ser = p.dumps(x);
Unpickler u = new Unpickler();
object y = u.loads(ser);
IDictionary dict = (IDictionary) y;
Assert.AreEqual(3, dict.Count);
Assert.AreEqual(34, dict["Age"]);
Assert.IsFalse(dict.Contains("__class__"));
}
public void testBeans()
{
Pickler p=new Pickler(false);
Unpickler pu=new Unpickler();
byte[] o;
Relative person=new Relative("Tupac",true);
o=p.dumps(person);
Hashtable map=(Hashtable) pu.loads(o);
Hashtable testmap=new Hashtable();
testmap.Add("Name","Tupac");
testmap.Add("Deceased",true);
testmap.Add("Relation","unspecified");
testmap.Add("__class__","Pyrolite.Tests.Pickle.PicklerTests+Relative");
AssertUtils.AssertEqual(testmap, map);
}
public static void Main(string[] args)
{
// going to pickle a c# datastructure
var map = new Dictionary<string, object>();
map["apple"] = 42;
map["microsoft"] = "hello";
var values = new List<double>();
values.AddRange(new double[] { 1.11, 2.22, 3.33, 4.44, 5.55} );
map["values"] = values;
// You can add many other types if you like. See the readme about the type mappings.
const string PickleFilename = "testpickle.dat";
Console.WriteLine("Writing pickle to '{0}'", PickleFilename);
var pickler = new Pickler(true);
using(FileStream fos = new FileStream(PickleFilename, FileMode.Create))
{
pickler.dump(map, fos);
}
Console.WriteLine("Done. Try unpickling it in python.\n");
Console.WriteLine("Reading a pickle created in python...");
// the following pickle was created in Python 3.4.
// it is this data: [1, 2, 3, (11, 12, 13), {'banana', 'grape', 'apple'}]
byte[] pythonpickle = new byte[] {128, 4, 149, 48, 0, 0, 0, 0, 0, 0, 0, 93, 148, 40, 75, 1, 75, 2, 75, 3, 75, 11, 75, 12, 75, 13, 135, 148, 143, 148, 40, 140, 6, 98, 97, 110, 97, 110, 97, 148, 140, 5, 103, 114, 97, 112, 101, 148, 140, 5, 97, 112, 112, 108, 101, 148, 144, 101, 46};
var unpickler = new Unpickler();
object result = unpickler.loads(pythonpickle);
Console.WriteLine("type: {0}", result.GetType());
var list = (ArrayList) result;
int integer1 = (int)list[0];
int integer2 = (int)list[1];
int integer3 = (int)list[2];
object[] tuple = (object[]) list[3];
HashSet<object> set = (HashSet<object>) list[4];
Console.WriteLine("1-3: integers: {0}, {1}, {2}", integer1, integer2, integer3);
Console.WriteLine("4: tuple: ({0}, {1}, {2})", tuple[0], tuple[1], tuple[2]);
Console.WriteLine("5: set: {0}", string.Join(",", set));
Console.WriteLine("\r\nEnter to exit:"); Console.ReadLine();
}
public static void Main(string[] args)
{
Unpickler u=new Unpickler();
byte[] data;
object result;
Console.WriteLine("here we go; 1");
data=PickleUtils.str2bytes("\u0080\u0002carray\narray\nq\u0000U\u0001iq\u0001]q\u0002\u0086q\u0003Rq\u0004.");
result=u.loads(data);
PrettyPrint.print(result);
Console.WriteLine("here we go; 2");
data=PickleUtils.str2bytes("\u0080\u0003carray\n_array_reconstructor\nq\u0000(carray\narray\nq\u0001X\u0001\u0000\u0000\u0000iq\u0002K\u0008C\u000c\u000f'\u0000\u0000\u00b8\"\u0000\u0000a\u001e\u0000\u0000q\u0003tq\u0004Rq\u0005.");
result=u.loads(data);
PrettyPrint.print(result);
Console.Write("Press any key to continue . . . ");
Console.ReadKey(true);
}
private static IEnumerable<dynamic> GetIterator(ISparkCLRSocket socket, string serializedMode)
{
Log("Serialized mode in GetIterator: " + serializedMode);
IFormatter formatter = new BinaryFormatter();
int messageLength;
do
{
messageLength = socket.ReadInt();
if (messageLength > 0)
{
byte[] buffer = socket.ReadBytes(messageLength);
switch (serializedMode)
{
case "String":
yield return SerDe.ToString(buffer);
break;
case "Row":
Unpickler unpickler = new Unpickler();
foreach (var item in (unpickler.loads(buffer) as object[]))
{
yield return item;
}
break;
case "Pair":
messageLength = socket.ReadInt();
byte[] value = socket.ReadBytes(messageLength);
yield return new KeyValuePair<byte[], byte[]>(buffer, value);
break;
case "Byte":
default:
var ms = new MemoryStream(buffer);
dynamic message = formatter.Deserialize(ms);
yield return message;
break;
}
}
} while (messageLength >= 0);
}
private static IEnumerable<dynamic> GetIterator(Stream s, string serializedMode)
{
Log("Serialized mode in GetIterator: " + serializedMode);
IFormatter formatter = new BinaryFormatter();
int messageLength;
while ((messageLength = SerDe.ReadInt(s)) != (int)SpecialLengths.END_OF_DATA_SECTION)
{
if (messageLength > 0 || serializedMode == "Pair")
{
byte[] buffer = messageLength > 0 ? SerDe.ReadBytes(s, messageLength) : null;
switch (serializedMode)
{
case "String":
yield return SerDe.ToString(buffer);
break;
case "Row":
Unpickler unpickler = new Unpickler();
foreach (var item in (unpickler.loads(buffer) as object[]))
{
yield return item;
}
break;
case "Pair":
messageLength = SerDe.ReadInt(s);
if (messageLength > 0)
{
yield return new KeyValuePair<byte[], byte[]>(buffer, SerDe.ReadBytes(s, messageLength));
}
break;
case "Byte":
default:
var ms = new MemoryStream(buffer);
dynamic message = formatter.Deserialize(ms);
yield return message;
break;
}
}
}
}
public void testBinint2WithObject()
{
Unpickler u=new Unpickler();
byte[] data=PickleUtils.str2bytes("\u0080\u0002cIgnore.Ignore\nIgnore\n)\u0081M\u0082#.");
int result=(int) u.loads(data);
Assert.AreEqual(9090,result);
}
public void testMemoizationSet()
{
var set = new HashSet<string>();
set.Add("a");
object[] array = new object[] {set, set};
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[]) u.loads(data);
Assert.AreEqual(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsInstanceOf(typeof(HashSet<object>), first);
Assert.IsInstanceOf(typeof(HashSet<object>), second);
Assert.AreSame(first, second); // both objects should be the same memoized object
HashSet<object> theSet = (HashSet<object>)second;
Assert.AreEqual(1, theSet.Count);
Assert.IsTrue(theSet.Contains("a"));
}
public void testMemoizationMap()
{
var map = new Dictionary<string,string>();
map.Add("key", "value");
object[] array = new object[] {map, map};
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[]) u.loads(data);
Assert.AreEqual(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsInstanceOf(typeof(Hashtable), first);
Assert.IsInstanceOf(typeof(Hashtable), second);
Assert.AreSame(first, second); // both objects should be the same memoized object
Hashtable theMap = (Hashtable) second;
Assert.AreEqual(1, theMap.Count);
Assert.AreEqual("value", theMap["key"]);
}
public void testMemoizationString()
{
string str = "a";
object[] array = new object[] {str, str};
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[]) u.loads(data);
Assert.AreEqual(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsInstanceOf(typeof(string), first);
Assert.IsInstanceOf(typeof(string), second);
Assert.AreSame(first, second); // both objects should be the same memoized object
str = (string) second;
Assert.AreEqual("a", str);
}
public void testBinint2WithObject()
{
Unpickler u=new Unpickler();
Unpickler.registerConstructor("Pyro4.core", "URI", new StringConstructor());
byte[] data=PickleUtils.str2bytes("\u0080\u0002cPyro4.core\nURI\n)\u0081M\u0082#.");
int result=(int) u.loads(data);
Assert.AreEqual(9090,result);
}
public void testClass()
{
Pickler p=new Pickler(false);
Unpickler pu=new Unpickler();
byte[] o;
Relative person=new Relative("Tupac",true, new int[] {3,4,5});
o=p.dumps(person);
Hashtable map=(Hashtable) pu.loads(o);
Assert.AreEqual(5, map.Count);
Assert.AreEqual("Pyrolite.Tests.Pickle.PicklerTests+Relative", map["__class__"]);
Assert.AreEqual("Tupac", map["Name"]);
Assert.AreEqual("unspecified", map["Relation"]);
Assert.AreEqual(true, map["Deceased"]);
Assert.AreEqual(new int[] {3,4,5}, map["Values"]);
}
public void testCustomPickler()
{
Pickler.registerCustomPickler(typeof(CustomClass), new CustomClassPickler());
CustomClass c=new CustomClass();
Pickler p=new Pickler(false);
byte[] ser = p.dumps(c);
Unpickler u = new Unpickler();
string x = (string) u.loads(ser);
Assert.AreEqual("customclassint=42", x);
}
public void testMemoization()
{
byte[] o;
Pickler p=new Pickler();
string reused = "reused";
string another = "another";
IList list=new ArrayList();
IList sublist = new ArrayList();
sublist.Add(reused);
sublist.Add(reused);
sublist.Add(another);
list.Add(reused);
list.Add(reused);
list.Add(another);
list.Add(sublist);
o=p.dumps(list);
Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01X\x07\x00\x00\x0000anotherq\x02]q\x03(h\x01h\x01h\x0002ee.", S(o));
Unpickler u = new Unpickler();
ArrayList data = (ArrayList) u.loads(o);
Assert.AreEqual(4, data.Count);
string s1 = (string) data[0];
string s2 = (string) data[1];
string s3 = (string) data[2];
data = (ArrayList) data[3];
string s4 = (string) data[0];
string s5 = (string) data[1];
string s6 = (string) data[2];
Assert.AreEqual("reused", s1);
Assert.AreEqual("another", s3);
Assert.AreSame(s1, s2);
Assert.AreSame(s3, s6);
Assert.AreSame(s1, s4);
Assert.AreSame(s1, s5);
}
public void testMemoizationRecursiveMemo()
{
byte[] o;
Pickler p=new Pickler();
// self-referencing list
string reused = "reused";
IList list=new ArrayList();
list.Add(reused);
list.Add(reused);
list.Add(list);
o=p.dumps(list);
Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01h\x0000e.", S(o));
Unpickler u = new Unpickler();
ArrayList data = (ArrayList) u.loads(o);
Assert.AreEqual(3, data.Count);
string s1 = (string) data[0];
string s2 = (string) data[1];
ArrayList data2 = (ArrayList) data[2];
Assert.AreEqual("reused", s1);
Assert.AreSame(s1, s2);
Assert.AreSame(data, data2);
Assert.AreSame(data[0], data2[0]);
// self-referencing hashtable
Hashtable h = new Hashtable();
h["myself"] = h;
o=p.dumps(h);
Assert.AreEqual("\x80\x02}q\x00(X\x06\x00\x00\x0000myselfq\x01h\x0000u.", S(o));
Hashtable h2 = (Hashtable) u.loads(o);
Assert.AreEqual(1, h2.Count);
Assert.AreSame(h2, h2["myself"]);
}
public void testSets()
{
byte[] o;
Pickler p=new Pickler(false);
Unpickler up=new Unpickler();
var intset=new HashSet<int>();
intset.Add(1);
intset.Add(2);
intset.Add(3);
o=p.dumps(intset);
HashSet<object> resultset=(HashSet<object>)up.loads(o);
AssertUtils.AssertEqual(intset, resultset);
HashSet<string> stringset=new HashSet<string>();
stringset.Add("A");
stringset.Add("B");
stringset.Add("C");
o=p.dumps(stringset);
resultset=(HashSet<object>)up.loads(o);
AssertUtils.AssertEqual(stringset, resultset);
}
public void testMappings()
{
byte[] o;
Pickler p=new Pickler(false);
Unpickler pu=new Unpickler();
var intmap=new Dictionary<int,int>();
intmap.Add(1, 11);
intmap.Add(2, 22);
intmap.Add(3, 33);
o=p.dumps(intmap);
Hashtable resultmap=(Hashtable)pu.loads(o);
AssertUtils.AssertEqual(intmap, resultmap);
var stringmap=new Dictionary<string,string>();
stringmap.Add("A", "1");
stringmap.Add("B", "2");
stringmap.Add("C", "3");
o=p.dumps(stringmap);
resultmap=(Hashtable)pu.loads(o);
AssertUtils.AssertEqual(stringmap, resultmap);
Hashtable table=new Hashtable();
table.Add(1,11);
table.Add(2,22);
table.Add(3,33);
o=p.dumps(table);
resultmap=(Hashtable)pu.loads(o);
AssertUtils.AssertEqual(table, resultmap);
}
public void testMemoizationTimeStuff()
{
TimeSpan delta = new TimeSpan(1,2,3);
DateTime time = new DateTime(2014,11,20,1,2,3);
object[] array = new object[] {delta, delta, time, time};
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[]) u.loads(data);
Assert.AreEqual(4, result.Length);
Assert.IsInstanceOf(typeof(TimeSpan), result[0]);
Assert.IsInstanceOf(typeof(TimeSpan), result[1]);
Assert.IsInstanceOf(typeof(DateTime), result[2]);
Assert.IsInstanceOf(typeof(DateTime), result[3]);
Assert.AreSame(result[0], result[1]); // both objects should be the same memoized object
Assert.AreSame(result[2], result[3]); // both objects should be the same memoized object
delta = (TimeSpan) result[1];
time = (DateTime) result[3];
Assert.AreEqual(new TimeSpan(1,2,3), delta);
Assert.AreEqual(new DateTime(2014,11,20,1,2,3), time);
}
public void testZeroToTwoFiveSix()
{
byte[] bytes=new byte[256];
for(int b=0; b<256; ++b) {
bytes[b]=(byte)b;
}
StringBuilder sb=new StringBuilder();
for(int i=0; i<256; ++i) {
sb.Append((char)i);
}
string str=sb.ToString();
Pickler p=new Pickler(false);
Unpickler u=new Unpickler();
MemoryStream bos=new MemoryStream(434);
bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2);
byte[] data=Encoding.Default.GetBytes("c__builtin__\nbytearray\n");
bos.Write(data,0,data.Length);
bos.WriteByte(Opcodes.BINUNICODE);
bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4);
byte[] utf8=Encoding.UTF8.GetBytes(str);
bos.Write(utf8,0,utf8.Length);
bos.WriteByte(Opcodes.BINUNICODE);
bos.Write(new byte[] {7,0,0,0},0,4);
data=Encoding.Default.GetBytes("latin-1");
bos.Write(data,0,data.Length);
bos.WriteByte(Opcodes.TUPLE2);
bos.WriteByte(Opcodes.REDUCE);
bos.WriteByte(Opcodes.STOP);
byte[] bytesresult=bos.ToArray();
byte[] output=p.dumps(bytes);
Assert.AreEqual(bytesresult, output);
Assert.AreEqual(bytes, (byte[])u.loads(output));
bos=new MemoryStream(434);
bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2);
bos.WriteByte(Opcodes.BINUNICODE);
bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4);
utf8=Encoding.UTF8.GetBytes(str);
bos.Write(utf8,0,utf8.Length);
bos.WriteByte(Opcodes.STOP);
bytesresult=bos.ToArray();
output=p.dumps(str);
Assert.AreEqual(bytesresult, output);
Assert.AreEqual(str, u.loads(output));
}
public void testDateTimeStringEscaping()
{
DateTime dt=new DateTime(2011, 10, 10, 9, 13, 10, 10);
Pickler p = new Pickler();
byte[] pickle = p.dumps(dt);
Unpickler u = new Unpickler();
DateTime dt2 = (DateTime) u.loads(pickle);
Assert.AreEqual(dt, dt2);
dt = new DateTime(2011, 10, 9, 13, 10, 9, 10);
dt2 = (DateTime) U("\u0080\u0002cdatetime\ndatetime\nq\u0000U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001\u0085q\u0002Rq\u0003."); // protocol 2
Assert.AreEqual(dt, dt2);
dt2 = (DateTime) U("cdatetime\ndatetime\nq\u0000(U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001tq\u0002Rq\u0003."); // protocol 1
Assert.AreEqual(dt, dt2);
dt2 = (DateTime) U("cdatetime\ndatetime\np0\n(S'\\x07\\xdb\\n\\t\\r\\n\\t\\x00'\\x10'\np1\ntp2\nRp3\n."); // protocol 0
Assert.AreEqual(dt, dt2);
}
public void setUp()
{
u=new Unpickler();
}
/// <summary>
/// Unpickles objects from byte[]
/// </summary>
/// <param name="buffer"></param>
/// <returns></returns>
internal static object[] GetUnpickledObjects(byte[] buffer)
{
var unpickler = new Unpickler(); //not making any assumptions about the implementation and hence not a class member
return (unpickler.loads(buffer) as object[]);
}
public void testUnpicklingPerformance()
{
Pickler pickler = new Pickler();
var myList = new List<string>();
for (int i = 0; i < 10; i++) {
myList.Add(i.ToString());
}
byte[] bytes = pickler.dumps(myList);
Unpickler unpickler = new Unpickler();
DateTime start = DateTime.Now;
for (int i = 0; i < 1000000; i++) {
unpickler.loads(bytes);
}
Console.WriteLine(DateTime.Now - start);
}
public void testDates()
{
Pickler p=new Pickler(false);
Unpickler u=new Unpickler();
DateTime date=new DateTime(2011,12,31,14,33,59);
byte[] o = p.dumps(date);
Object unpickled=u.loads(o);
Assert.AreEqual(date, unpickled);
date=new DateTime(2011,12,31,14,33,59,456);
o=p.dumps(date);
unpickled=u.loads(o);
Assert.AreEqual(date, unpickled);
}
public void TestWorkerWithBytesDeserializedModeAndRowSerializedMode()
{
const int expectedCount = 100;
Process worker;
var CSharpRDD_SocketServer = CreateServer(out worker);
using (var serverSocket = CSharpRDD_SocketServer.Accept())
using (var s = serverSocket.GetStream())
{
WritePayloadHeaderToWorker(s);
byte[] command = SparkContext.BuildCommand(new CSharpWorkerFunc((pid, iter) => iter), SerializedMode.Byte, SerializedMode.Row);
SerDe.Write(s, command.Length);
SerDe.Write(s, command);
var formatter = new BinaryFormatter();
for (int i = 0; i < expectedCount; i++)
{
var ms = new MemoryStream();
formatter.Serialize(ms, i);
var buffer = ms.ToArray();
SerDe.Write(s, buffer.Length);
SerDe.Write(s, ms.ToArray());
}
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
s.Flush();
lock (syncLock)
{
Console.WriteLine(output);
}
int count = 0;
Unpickler unpickler = new Unpickler();
foreach (var bytes in ReadWorker(s))
{
var objects = unpickler.loads(bytes) as ArrayList;
Assert.IsNotNull(objects);
Assert.IsTrue(objects.Count == 1);
Assert.AreEqual(count++, (int)objects[0]);
}
Assert.AreEqual(expectedCount, count);
}
AssertWorker(worker);
CSharpRDD_SocketServer.Close();
}
public void testTimes()
{
Pickler p=new Pickler(false);
Unpickler u=new Unpickler();
TimeSpan ts=new TimeSpan(2, 0, 0, 7000, 456);
byte[] o = p.dumps(ts);
object unpickled=u.loads(o);
Assert.AreEqual(ts,unpickled);
Assert.AreEqual(B("cdatetime\ntimedelta\nK\u0002MX\u001bJ@\u00f5\u0006\u0000\u0087R"), o);
}
