public void testMemoizationString()
{
string str = "a";
object[] array = new object[] { str, str };
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[])u.loads(data);
Assert.AreEqual(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsInstanceOf(typeof(string), first);
Assert.IsInstanceOf(typeof(string), second);
Assert.AreSame(first, second); // both objects should be the same memoized object
str = (string)second;
Assert.AreEqual("a", str);
}
public void testMemoizationCollection()
{
ICollection <string> list = new List <string>();
list.Add("a");
object[] array = new object[] { list, list };
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[])u.loads(data);
Assert.AreEqual(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsInstanceOf(typeof(ArrayList), first);
Assert.IsInstanceOf(typeof(ArrayList), second);
Assert.AreSame(first, second); // both objects should be the same memoized object
ArrayList theList = (ArrayList)second;
Assert.AreEqual(1, theList.Count);
Assert.IsTrue(theList.Contains("a"));
}
public void testSets()
{
byte[] o;
Pickler p = new Pickler(false);
Unpickler up = new Unpickler();
var intset = new HashSet <int>();
intset.Add(1);
intset.Add(2);
intset.Add(3);
o = p.dumps(intset);
HashSet <object> resultset = (HashSet <object>)up.loads(o);
AssertUtils.AssertEqual(intset, resultset);
HashSet <string> stringset = new HashSet <string>();
stringset.Add("A");
stringset.Add("B");
stringset.Add("C");
o = p.dumps(stringset);
resultset = (HashSet <object>)up.loads(o);
AssertUtils.AssertEqual(stringset, resultset);
}
public void testMemoizationTimeStuff()
{
TimeSpan delta = new TimeSpan(1, 2, 3);
DateTime time = new DateTime(2014, 11, 20, 1, 2, 3);
object[] array = new object[] { delta, delta, time, time };
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[])u.loads(data);
Assert.AreEqual(4, result.Length);
Assert.IsInstanceOf(typeof(TimeSpan), result[0]);
Assert.IsInstanceOf(typeof(TimeSpan), result[1]);
Assert.IsInstanceOf(typeof(DateTime), result[2]);
Assert.IsInstanceOf(typeof(DateTime), result[3]);
Assert.AreSame(result[0], result[1]); // both objects should be the same memoized object
Assert.AreSame(result[2], result[3]); // both objects should be the same memoized object
delta = (TimeSpan)result[1];
time = (DateTime)result[3];
Assert.AreEqual(new TimeSpan(1, 2, 3), delta);
Assert.AreEqual(new DateTime(2014, 11, 20, 1, 2, 3), time);
}
public void testMemoizationSet()
{
var set = new HashSet <string>();
set.Add("a");
object[] array = new object[] { set, set };
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[])u.loads(data);
Assert.AreEqual(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsInstanceOf(typeof(HashSet <object>), first);
Assert.IsInstanceOf(typeof(HashSet <object>), second);
Assert.AreSame(first, second); // both objects should be the same memoized object
HashSet <object> theSet = (HashSet <object>)second;
Assert.AreEqual(1, theSet.Count);
Assert.IsTrue(theSet.Contains("a"));
}
public void testMemoizationMap()
{
var map = new Dictionary <string, string>();
map.Add("key", "value");
object[] array = new object[] { map, map };
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[])u.loads(data);
Assert.AreEqual(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsInstanceOf(typeof(Hashtable), first);
Assert.IsInstanceOf(typeof(Hashtable), second);
Assert.AreSame(first, second); // both objects should be the same memoized object
Hashtable theMap = (Hashtable)second;
Assert.AreEqual(1, theMap.Count);
Assert.AreEqual("value", theMap["key"]);
}
public void testMappings()
{
byte[] o;
Pickler p = new Pickler(false);
Unpickler pu = new Unpickler();
var intmap = new Dictionary <int, int>();
intmap.Add(1, 11);
intmap.Add(2, 22);
intmap.Add(3, 33);
o = p.dumps(intmap);
Hashtable resultmap = (Hashtable)pu.loads(o);
AssertUtils.AssertEqual(intmap, resultmap);
var stringmap = new Dictionary <string, string>();
stringmap.Add("A", "1");
stringmap.Add("B", "2");
stringmap.Add("C", "3");
o = p.dumps(stringmap);
resultmap = (Hashtable)pu.loads(o);
AssertUtils.AssertEqual(stringmap, resultmap);
Hashtable table = new Hashtable();
table.Add(1, 11);
table.Add(2, 22);
table.Add(3, 33);
o = p.dumps(table);
resultmap = (Hashtable)pu.loads(o);
AssertUtils.AssertEqual(table, resultmap);
}
object U(byte[] data)
{
Unpickler u=new Unpickler();
object o=u.loads(data);
u.close();
return o;
}
public override object deserializeData(byte[] data)
{
using (var u = new Unpickler())
{
return(u.loads(data));
}
}
private static object U(byte[] data)
{
using (Unpickler u = new Unpickler())
{
return(u.loads(data));
}
}
public void TestMemoizationMap()
{
var map = new Dictionary <string, string> {
{ "key", "value" }
};
object[] array = { map, map };
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[])u.loads(data);
Assert.Equal(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsType <Hashtable>(first);
Assert.IsType <Hashtable>(second);
Assert.Same(first, second); // both objects should be the same memoized object
Hashtable theMap = (Hashtable)second;
Assert.Single(theMap);
Assert.Equal("value", theMap["key"]);
}
public void TestMemoizationSet()
{
var set = new HashSet <string> {
"a"
};
object[] array = { set, set };
Pickler p = new Pickler(true);
byte[] data = p.dumps(array);
Assert.Contains(Opcodes.BINPUT, data); // check that memoization was done
Unpickler u = new Unpickler();
object[] result = (object[])u.loads(data);
Assert.Equal(2, result.Length);
object first = result[0];
object second = result[1];
Assert.IsType <HashSet <object> >(first);
Assert.IsType <HashSet <object> >(second);
Assert.Same(first, second); // both objects should be the same memoized object
HashSet <object> theSet = (HashSet <object>)second;
Assert.Single(theSet);
Assert.Contains("a", theSet);
}
public void TestMappings()
{
Pickler p = new Pickler(false);
Unpickler pu = new Unpickler();
var intmap = new Dictionary <int, int> {
{ 1, 11 }, { 2, 22 }, { 3, 33 }
};
var o = p.dumps(intmap);
Hashtable resultmap = (Hashtable)pu.loads(o);
AssertUtils.AssertEqual(intmap, resultmap);
var stringmap = new Dictionary <string, string> {
{ "A", "1" }, { "B", "2" }, { "C", "3" }
};
o = p.dumps(stringmap);
resultmap = (Hashtable)pu.loads(o);
AssertUtils.AssertEqual(stringmap, resultmap);
Hashtable table = new Hashtable {
{ 1, 11 }, { 2, 22 }, { 3, 33 }
};
o = p.dumps(table);
resultmap = (Hashtable)pu.loads(o);
AssertUtils.AssertEqual(table, resultmap);
}
public void TestBinint2WithObject()
{
Unpickler u = new Unpickler();
var data = PickleUtils.str2bytes("\u0080\u0002cIgnore.Ignore\nIgnore\n)\u0081M\u0082#.");
int result = (int)u.loads(data);
Assert.Equal(9090, result);
}
/// <summary>
/// Unpickles objects from byte[]
/// </summary>
/// <param name="buffer"></param>
/// <returns></returns>
internal static object[] GetUnpickledObjects(byte[] buffer)
{
var unpickler = new Unpickler(); //not making any assumptions about the implementation and hence not a class member
var unpickledItems = unpickler.loads(buffer);
Debug.Assert(unpickledItems != null);
return(unpickledItems as object[]);
}
private static object U(byte[] data)
{
Unpickler u = new Unpickler();
object o = u.loads(data);
u.close();
return(o);
}
public void TestZeroToTwoFiveSix()
{
var bytes = new byte[256];
for (int b = 0; b < 256; ++b)
{
bytes[b] = (byte)b;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 256; ++i)
{
sb.Append((char)i);
}
string str = sb.ToString();
Pickler p = new Pickler(false);
Unpickler u = new Unpickler();
MemoryStream bos = new MemoryStream(434);
bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2);
var data = Encoding.Default.GetBytes("c__builtin__\nbytearray\n");
bos.Write(data, 0, data.Length);
bos.WriteByte(Opcodes.BINUNICODE);
bos.Write(new byte[] { 0x80, 0x01, 0x00, 0x00 }, 0, 4);
var utf8 = Encoding.UTF8.GetBytes(str);
bos.Write(utf8, 0, utf8.Length);
bos.WriteByte(Opcodes.BINUNICODE);
bos.Write(new byte[] { 7, 0, 0, 0 }, 0, 4);
data = Encoding.Default.GetBytes("latin-1");
bos.Write(data, 0, data.Length);
bos.WriteByte(Opcodes.TUPLE2);
bos.WriteByte(Opcodes.REDUCE);
bos.WriteByte(Opcodes.STOP);
var bytesresult = bos.ToArray();
var output = p.dumps(bytes);
Assert.Equal(bytesresult, output);
Assert.Equal(bytes, (byte[])u.loads(output));
bos = new MemoryStream(434);
bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2);
bos.WriteByte(Opcodes.BINUNICODE);
bos.Write(new byte[] { 0x80, 0x01, 0x00, 0x00 }, 0, 4);
utf8 = Encoding.UTF8.GetBytes(str);
bos.Write(utf8, 0, utf8.Length);
bos.WriteByte(Opcodes.STOP);
bytesresult = bos.ToArray();
output = p.dumps(str);
Assert.Equal(bytesresult, output);
Assert.Equal(str, u.loads(output));
}
public void TestBYTEARRAY8()
{
// BYTEARRAY8 = 0x96 (pickle protocol 5)
Unpickler u = new Unpickler();
byte[] data = PickleUtils.str2bytes("\u0080\u0005\u0095\u000e\u0000\u0000\u0000\u0000\u0000\u0000\u0090\u0096\u0003\u0000\u0000\u0000\u0000\u0000\u0000\u0000abc\u0094.");
byte[] result = (byte[])u.loads(data);
Assert.Equal(new byte[] { 97, 98, 99 }, result);
}
/// <summary>
/// Unpickles objects from byte[].
/// </summary>
/// <param name="s">Pickled byte stream</param>
/// <returns>Unpicked objects</returns>
internal static object[] GetUnpickledObjects(Stream s)
{
// Not making any assumptions about the implementation and hence not a class member.
var unpickler = new Unpickler();
var unpickledItems = unpickler.load(s);
Debug.Assert(unpickledItems != null);
return(unpickledItems as object[]);
}
static PythonSerDe()
{
// Custom picklers used in PySpark implementation.
// Refer to spark/python/pyspark/sql/types.py.
Unpickler.registerConstructor(
"pyspark.sql.types", "_parse_datatype_json_string", new StringConstructor());
Unpickler.registerConstructor(
"pyspark.sql.types", "_create_row_inbound_converter", new RowConstructor());
}
public void testBinint2WithObject()
{
Unpickler u = new Unpickler();
Unpickler.registerConstructor("Pyro4.core", "URI", new StringConstructor());
byte[] data = PickleUtils.str2bytes("\u0080\u0002cPyro4.core\nURI\n)\u0081M\u0082#.");
int result = (int)u.loads(data);
Assert.AreEqual(9090, result);
}
public void TestBUILD()
{
//BUILD = b'b' # call __setstate__ or __dict__.update()
Unpickler.registerConstructor("unittest", "Thingy", new ThingyConstructor());
// create a thing with initial value for the field 'a',
// the use BUILD to __setstate__() it with something else ('foo').
ThingyWithSetstate thingy = (ThingyWithSetstate)U("cunittest\nThingy\n(V123\ntR}S'a'\nS'foo'\nsb.");
Assert.Equal("foo", thingy.a);
}
private void UpdateDossierFile(FileInfo fi)
{
using (Unpickler unpickler = new Unpickler())
{
using (FileStream fs = fi.OpenRead())
{
DossierData dossier = ParseDossierRawData(fi.Name, unpickler.load(fs));
DossierDB.UpdateDossierData(fi.Name, fi.LastWriteTimeUtc.ToUnixTime(), dossier);
}
}
}
public void TestProtocol5bytearray()
{
// ["string", bytearray(b'irmen')] -> produces a protocol 5 pickle with opcode BYTEARRAY8
Unpickler u = new Unpickler();
byte[] data = PickleUtils.str2bytes("\u0080\u0005\u0095\u001d\u0000\u0000\u0000\u0000\u0000\u0000\u0000]\u0094(\u008c\u0006string\u0094\u0096\u0005\u0000\u0000\u0000\u0000\u0000\u0000\u0000irmen\u0094e.");
ArrayList result = (ArrayList)u.loads(data);
Assert.Equal(2, result.Count);
Assert.Equal("string", result[0]);
Assert.Equal(new byte[] { 105, 114, 109, 101, 110 }, result[1]);
}
public void TestUnpickleCustomClass()
{
byte[] pickled = { 128, 2, 99, 95, 95, 109, 97, 105, 110, 95, 95, 10, 67, 117, 115, 116, 111, 109, 67, 108, 97, 122, 122, 10, 113, 0, 41, 129, 113, 1, 125, 113, 2, 40, 85, 3, 97, 103, 101, 113, 3, 75, 34, 85, 6, 118, 97, 108, 117, 101, 115, 113, 4, 93, 113, 5, 40, 75, 1, 75, 2, 75, 3, 101, 85, 4, 110, 97, 109, 101, 113, 6, 85, 5, 72, 97, 114, 114, 121, 113, 7, 117, 98, 46 };
Unpickler.registerConstructor("__main__", "CustomClazz", new CustomClazzConstructor());
CustomClazz o = (CustomClazz)U(pickled);
Assert.Equal("Harry", o.Name);
Assert.Equal(34, o.Age);
Assert.Equal(new ArrayList {
1, 2, 3
}, o.Values);
}
public void testCustomPickler()
{
Pickler.registerCustomPickler(typeof(CustomClass), new CustomClassPickler());
CustomClass c = new CustomClass();
Pickler p = new Pickler(false);
byte[] ser = p.dumps(c);
Unpickler u = new Unpickler();
string x = (string)u.loads(ser);
Assert.AreEqual("customclassint=42", x);
}
public void testTimes()
{
Pickler p = new Pickler(false);
Unpickler u = new Unpickler();
TimeSpan ts = new TimeSpan(2, 0, 0, 7000, 456);
byte[] o = p.dumps(ts);
object unpickled = u.loads(o);
Assert.AreEqual(ts, unpickled);
Assert.AreEqual(B("cdatetime\ntimedelta\nK\u0002MX\u001bJ@\u00f5\u0006\u0000\u0087R"), o);
}
static PythonSerDe()
{
// Custom picklers used in PySpark implementation.
// Refer to spark/python/pyspark/sql/types.py.
Unpickler.registerConstructor(
"pyspark.sql.types", "_parse_datatype_json_string", new StringConstructor());
s_rowConstructor = new RowConstructor();
Unpickler.registerConstructor(
"pyspark.sql.types", "_create_row_inbound_converter", s_rowConstructor);
// Register custom pickler for GenericRow objects.
Pickler.registerCustomPickler(typeof(GenericRow), new GenericRowPickler());
}
public static void Main(string[] args)
{
// going to pickle a c# datastructure
var map = new Dictionary <string, object>();
map["apple"] = 42;
map["microsoft"] = "hello";
var values = new List <double>();
values.AddRange(new double[] { 1.11, 2.22, 3.33, 4.44, 5.55 });
map["values"] = values;
// You can add many other types if you like. See the readme about the type mappings.
const string PickleFilename = "testpickle.dat";
Console.WriteLine("Writing pickle to '{0}'", PickleFilename);
var pickler = new Pickler(true);
using (FileStream fos = new FileStream(PickleFilename, FileMode.Create))
{
pickler.dump(map, fos);
}
Console.WriteLine("Done. Try unpickling it in python.\n");
Console.WriteLine("Reading a pickle created in python...");
// the following pickle was created in Python 3.4.
// it is this data: [1, 2, 3, (11, 12, 13), {'banana', 'grape', 'apple'}]
byte[] pythonpickle = new byte[] { 128, 4, 149, 48, 0, 0, 0, 0, 0, 0, 0, 93, 148, 40, 75, 1, 75, 2, 75, 3, 75, 11, 75, 12, 75, 13, 135, 148, 143, 148, 40, 140, 6, 98, 97, 110, 97, 110, 97, 148, 140, 5, 103, 114, 97, 112, 101, 148, 140, 5, 97, 112, 112, 108, 101, 148, 144, 101, 46 };
var unpickler = new Unpickler();
object result = unpickler.loads(pythonpickle);
Console.WriteLine("type: {0}", result.GetType());
var list = (ArrayList)result;
int integer1 = (int)list[0];
int integer2 = (int)list[1];
int integer3 = (int)list[2];
object[] tuple = (object[])list[3];
HashSet <object> set = (HashSet <object>)list[4];
Console.WriteLine("1-3: integers: {0}, {1}, {2}", integer1, integer2, integer3);
Console.WriteLine("4: tuple: ({0}, {1}, {2})", tuple[0], tuple[1], tuple[2]);
Console.WriteLine("5: set: {0}", string.Join(",", set));
Console.WriteLine("\r\nEnter to exit:"); Console.ReadLine();
}
public void TestWorkerWithBytesDeserializedModeAndRowSerializedMode()
{
const int expectedCount = 100;
Process worker;
var CSharpRDD_SocketServer = CreateServer(out worker);
using (var serverSocket = CSharpRDD_SocketServer.Accept())
using (var s = serverSocket.GetStream())
{
WritePayloadHeaderToWorker(s);
byte[] command = SparkContext.BuildCommand(new CSharpWorkerFunc((pid, iter) => iter), SerializedMode.Byte, SerializedMode.Row);
SerDe.Write(s, command.Length);
SerDe.Write(s, command);
var formatter = new BinaryFormatter();
for (int i = 0; i < expectedCount; i++)
{
var ms = new MemoryStream();
formatter.Serialize(ms, i);
var buffer = ms.ToArray();
SerDe.Write(s, buffer.Length);
SerDe.Write(s, ms.ToArray());
}
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
s.Flush();
lock (syncLock)
{
Console.WriteLine(output);
}
int count = 0;
Unpickler unpickler = new Unpickler();
foreach (var bytes in ReadWorker(s))
{
var objects = unpickler.loads(bytes) as ArrayList;
Assert.IsNotNull(objects);
Assert.IsTrue(objects.Count == 1);
Assert.AreEqual(count++, (int)objects[0]);
}
Assert.AreEqual(expectedCount, count);
}
AssertWorker(worker);
CSharpRDD_SocketServer.Close();
}