Example #1
1
        public void pickle(object o, Stream stream, Pickler currentPickler)
        {
            if (o.Equals(this))
            {
                SerDe.Write(stream, Opcodes.GLOBAL);
                SerDe.Write(stream, Encoding.UTF8.GetBytes(module + "\n" + "_create_row_inbound_converter" + "\n"));
            }
            else
            {
                var row = o as Row;
                if (row == null)
                {
                    throw new InvalidOperationException(this.GetType().Name + " only accepts 'Row' type objects.");
                }

                currentPickler.save(this);
                currentPickler.save(row.GetSchema());
                SerDe.Write(stream, Opcodes.TUPLE1);
                SerDe.Write(stream, Opcodes.REDUCE);

                SerDe.Write(stream, Opcodes.MARK);

                var i = 0;
                while (i < row.Size())
                {
                    currentPickler.save(row.Get(i));
                    i++;
                }

                SerDe.Write(stream, Opcodes.TUPLE);
                SerDe.Write(stream, Opcodes.REDUCE);
            }
        }
Example #2
0
        public void testArrays()
        {
            Pickler p = new Pickler(false);
            byte[] o;
            o=p.dumps(new string[] {});
            Assert.AreEqual(B(")"), o);
            o=p.dumps(new string[] {"abc"});
            Assert.AreEqual(B("X\u0003\u0000\u0000\u0000abc\u0085"), o);
            o=p.dumps(new string[] {"abc","def"});
            Assert.AreEqual(B("X\u0003\u0000\u0000\u0000abcX\u0003\u0000\u0000\u0000def\u0086"), o);
            o=p.dumps(new string[] {"abc","def","ghi"});
            Assert.AreEqual(B("X\u0003\u0000\u0000\u0000abcX\u0003\u0000\u0000\u0000defX\u0003\u0000\u0000\u0000ghi\u0087"), o);
            o=p.dumps(new string[] {"abc","def","ghi","jkl"});
            Assert.AreEqual(B("(X\u0003\u0000\u0000\u0000abcX\u0003\u0000\u0000\u0000defX\u0003\u0000\u0000\u0000ghiX\u0003\u0000\u0000\u0000jklt"), o);

            o=p.dumps(new char[] {'A','B','C'});
            Assert.AreEqual(B("X\u0003\u0000\u0000\u0000ABC"), o);

            o=p.dumps(new bool[] {true,false,true});
            Assert.AreEqual(B("\u0088\u0089\u0088\u0087"), o);

            o=p.dumps(new byte[] {1,2,3});
            Assert.AreEqual(B("c__builtin__\nbytearray\nX\u0003\u0000\u0000\u0000\u0001\u0002\u0003X\u0007\u0000\u0000\u0000latin-1\u0086R"), o);

            o=p.dumps(new int[] {1,2,3});
            Assert.AreEqual(B("carray\narray\nU\u0001i](K\u0001K\u0002K\u0003e\u0086R"), o);

            o=p.dumps(new double[] {1.1,2.2,3.3});
            Assert.AreEqual(B("carray\narray\nU\u0001d](G?\u00f1\u0099\u0099\u0099\u0099\u0099\u009aG@\u0001\u0099\u0099\u0099\u0099\u0099\u009aG@\nffffffe\u0086R"), o);
        }
        private int recurse;                   // recursion level

        internal PicklerImplementation(T output, Pickler pickler, bool useMemo)
        {
            this.output  = output;
            this.pickler = pickler;
            memo         = useMemo ? new Dictionary <object, int>() : null;
            recurse      = 0;
        }
Example #4
0
 public override byte[] serializeData(object obj)
 {
     using(var p=new Pickler())
     {
         return p.dumps(obj);
     }
 }
 public void testPickleUnpickleProxy()
 {
     PyroProxy proxy=new PyroProxy("hostname",9999,"objectid");
     Pickler p=new Pickler();
     byte[] pickled_proxy=p.dumps(proxy);
     object result=U(pickled_proxy);
     Assert.IsInstanceOfType(typeof(System.Collections.Hashtable), result); // proxy objects cannot be properly pickled and are pickled as bean, hence HashMap
 }
Example #6
0
 public override byte[] serializeCall(string objectId, string method, object[] vargs, IDictionary<string, object> kwargs)
 {
     using(var p=new Pickler())
     {
         object[] invokeparams = new object[] {objectId, method, vargs, kwargs};
         return p.dumps(invokeparams);
     }
 }
Example #7
0
        public void testAnonType()
        {
            var x = new { Name="Harry", Country="UK", Age=34 };
            Pickler p = new Pickler();
            byte[] ser = p.dumps(x);

            Unpickler u = new Unpickler();
            object y = u.loads(ser);
            IDictionary dict = (IDictionary) y;
            Assert.AreEqual(3, dict.Count);
            Assert.AreEqual(34, dict["Age"]);
            Assert.IsFalse(dict.Contains("__class__"));
        }
        public void testPickleUnpickleURI()
        {
            PyroURI uri=new PyroURI("PYRO:test@localhost:9999");
            Pickler p=new Pickler();
            byte[] pickled_uri=p.dumps(uri);
            PyroURI uri2=(PyroURI) U(pickled_uri);
            Assert.AreEqual(uri,uri2);

            uri=new PyroURI();
            pickled_uri=p.dumps(uri);
            uri2=(PyroURI) U(pickled_uri);
            Assert.AreEqual(uri,uri2);
        }
Example #9
0
        public void pickle(object o, Stream stream, Pickler currentPickler)
        {
            var schema = o as StructType;
            if (schema == null)
            {
                throw new InvalidOperationException(this.GetType().Name + " only accepts 'StructType' type objects.");
            }

            SerDe.Write(stream, Opcodes.GLOBAL);
            SerDe.Write(stream, Encoding.UTF8.GetBytes(module + "\n" + "_parse_datatype_json_string" + "\n"));
            currentPickler.save(schema.Json);
            SerDe.Write(stream, Opcodes.TUPLE1);
            SerDe.Write(stream, Opcodes.REDUCE);
        }
 public void testBeans()
 {
     Pickler p=new Pickler(false);
     Unpickler pu=new Unpickler();
     byte[] o;
     Relative person=new Relative("Tupac",true);
     o=p.dumps(person);
     Hashtable map=(Hashtable) pu.loads(o);
     Hashtable testmap=new Hashtable();
     testmap.Add("Name","Tupac");
     testmap.Add("Deceased",true);
     testmap.Add("Relation","unspecified");
     testmap.Add("__class__","Pyrolite.Tests.Pickle.PicklerTests+Relative");
     AssertUtils.AssertEqual(testmap, map);
 }
Example #11
0
	public void pickle(object o, Stream outs, Pickler currentPickler) {
		PyroURI uri = (PyroURI) o;
		outs.WriteByte(Opcodes.GLOBAL);
		byte[] output=Encoding.Default.GetBytes("Pyro4.core\nURI\n");
		outs.Write(output,0,output.Length);
		outs.WriteByte(Opcodes.EMPTY_TUPLE);
		outs.WriteByte(Opcodes.NEWOBJ);
		outs.WriteByte(Opcodes.MARK);
		currentPickler.save(uri.protocol);
		currentPickler.save(uri.objectid);
		currentPickler.save(null);
		currentPickler.save(uri.host);
		currentPickler.save(uri.port);
		outs.WriteByte(Opcodes.TUPLE);
		outs.WriteByte(Opcodes.BUILD);
	}
Example #12
0
	public void pickle(object o, Stream outs, Pickler currentPickler) {
		PyroProxy proxy = (PyroProxy) o;
		outs.WriteByte(Opcodes.GLOBAL);
		byte[] output=Encoding.Default.GetBytes("Pyro4.core\nProxy\n");
		outs.Write(output,0,output.Length);
		outs.WriteByte(Opcodes.EMPTY_TUPLE);
		outs.WriteByte(Opcodes.NEWOBJ);
		
		// parameters are: pyroUri, pyroOneway(hashset), pyroTimeout
		object[] args = new object[] {   
			new PyroURI(proxy.objectid, proxy.hostname, proxy.port),
			new HashSet<object>(),
			0.0
		};
		currentPickler.save(args);
		outs.WriteByte(Opcodes.BUILD);
	}
Example #13
0
        public static void Main(string[] args)
        {
            // going to pickle a c# datastructure

            var map = new Dictionary<string, object>();
            map["apple"] = 42;
            map["microsoft"] = "hello";
            var values = new List<double>();
            values.AddRange(new double[] { 1.11, 2.22, 3.33, 4.44, 5.55} );
            map["values"] = values;
            // You can add many other types if you like. See the readme about the type mappings.

            const string PickleFilename = "testpickle.dat";

            Console.WriteLine("Writing pickle to '{0}'", PickleFilename);

            var pickler = new Pickler(true);
            using(FileStream fos = new FileStream(PickleFilename, FileMode.Create))
            {
                pickler.dump(map, fos);
            }

            Console.WriteLine("Done. Try unpickling it in python.\n");

            Console.WriteLine("Reading a pickle created in python...");

            // the following pickle was created in Python 3.4.
            // it is this data:     [1, 2, 3, (11, 12, 13), {'banana', 'grape', 'apple'}]
            byte[] pythonpickle = new byte[]  {128, 4, 149, 48, 0, 0, 0, 0, 0, 0, 0, 93, 148, 40, 75, 1, 75, 2, 75, 3, 75, 11, 75, 12, 75, 13, 135, 148, 143, 148, 40, 140, 6, 98, 97, 110, 97, 110, 97, 148, 140, 5, 103, 114, 97, 112, 101, 148, 140, 5, 97, 112, 112, 108, 101, 148, 144, 101, 46};
            var unpickler = new Unpickler();
            object result = unpickler.loads(pythonpickle);

            Console.WriteLine("type: {0}", result.GetType());
            var list = (ArrayList) result;
            int integer1 = (int)list[0];
            int integer2 = (int)list[1];
            int integer3 = (int)list[2];
            object[] tuple = (object[]) list[3];
            HashSet<object> set = (HashSet<object>) list[4];
            Console.WriteLine("1-3: integers: {0}, {1}, {2}", integer1, integer2, integer3);
            Console.WriteLine("4: tuple: ({0}, {1}, {2})", tuple[0], tuple[1], tuple[2]);
            Console.WriteLine("5: set: {0}", string.Join(",", set));

            Console.WriteLine("\r\nEnter to exit:"); Console.ReadLine();
        }
Example #14
0
	public void pickle(object o, Stream outs, Pickler currentPickler) {
		PyroException error = (PyroException) o;
		outs.WriteByte(Opcodes.GLOBAL);
		byte[] output=Encoding.Default.GetBytes("Pyro4.errors\nPyroError\n");
		outs.Write(output,0,output.Length);
		object[] args = new object[] { error.Message };
		currentPickler.save(args);
		outs.WriteByte(Opcodes.REDUCE);
		
		if(!string.IsNullOrEmpty(error._pyroTraceback))
		{
			// add _pyroTraceback attribute to the output
			Hashtable tb = new Hashtable();
			tb["_pyroTraceback"] = new string[]{ error._pyroTraceback };		// transform single string back into list
			currentPickler.save(tb);
			outs.WriteByte(Opcodes.BUILD);
		}
	}
Example #15
0
	public void testSinglePrimitives() {
		// protocol level 2
		Pickler p=new Pickler(false);
		byte[] o=p.dumps(null);	// none
		Assert.AreEqual(B("N"), o); 
		o=p.dumps('@');  // char --> string
		Assert.AreEqual(B("X\u0001\u0000\u0000\u0000@"), o);
		o=p.dumps(true);	// bool
		Assert.AreEqual(B("\u0088"), o);
		o=p.dumps("hello");      // unicode string
		Assert.AreEqual(B("X\u0005\u0000\u0000\u0000hello"), o);
		o=p.dumps("hello\u20ac");      // unicode string with non ascii
		Assert.AreEqual(B("X\u0008\u0000\u0000\u0000hello\u00e2\u0082\u00ac"), o);
		o=p.dumps((byte)'@');
		Assert.AreEqual(B("K@"), o);
		o=p.dumps((sbyte)-40);
		Assert.AreEqual( B(new byte[]{(byte)'J',0xd8,0xff,0xff,0xff}), o);
		o=p.dumps((short)-0x1234);
		Assert.AreEqual(B("J\u00cc\u00ed\u00ff\u00ff"), o);
		o=p.dumps((ushort)0xf234);
		Assert.AreEqual(B("M\u0034\u00f2"), o);
		o=p.dumps((int)-0x12345678);
		Assert.AreEqual(B("J\u0088\u00a9\u00cb\u00ed"), o);
		o=p.dumps((uint)0x12345678);
		Assert.AreEqual(B(new byte[]{(byte)'J', 0x78, 0x56, 0x34, 0x12}), o);
		o=p.dumps((uint)0xf2345678);
		Assert.AreEqual(B("I4063516280\n"), o);
		o=p.dumps((long)0x12345678abcdefL);
		Assert.AreEqual(B("I5124095577148911\n"), o);
		o=p.dumps(1234.5678d);
		Assert.AreEqual(B(new byte[] {(byte)'G',0x40,0x93,0x4a,0x45,0x6d,0x5c,0xfa,0xad}), o);
		o=p.dumps(1234.5f);
		Assert.AreEqual(B(new byte[] {(byte)'G',0x40,0x93,0x4a,0,0,0,0,0}), o);
		o=p.dumps(1234.9876543210987654321m);
		Assert.AreEqual(B("cdecimal\nDecimal\nX\u0018\u0000\u0000\u00001234.9876543210987654321\u0085R"), o);
		
		DayEnum day=DayEnum.WEDNESDAY;
		o=p.dumps(day);	// enum is returned as just a string representing the value
		Assert.AreEqual(B("X\u0009\u0000\u0000\u0000WEDNESDAY"),o);
	}
Example #16
0
	public void testClass() 
	{
		Pickler p=new Pickler(false);
		Unpickler pu=new Unpickler();
		byte[] o;
		Relative person=new Relative("Tupac",true, new int[] {3,4,5});
		o=p.dumps(person);
		Hashtable map=(Hashtable) pu.loads(o);
		
		Assert.AreEqual(5, map.Count);
		Assert.AreEqual("Pyrolite.Tests.Pickle.PicklerTests+Relative", map["__class__"]);
		Assert.AreEqual("Tupac", map["Name"]);
		Assert.AreEqual("unspecified", map["Relation"]);
		Assert.AreEqual(true, map["Deceased"]);
		Assert.AreEqual(new int[] {3,4,5}, map["Values"]);
	}
Example #17
0
        public void pickle(object o, Stream outs, Pickler currentPickler)
        {
            PyroProxy proxy = (PyroProxy) o;
            outs.WriteByte(Opcodes.GLOBAL);
            byte[] output=Encoding.Default.GetBytes("Pyro4.core\nProxy\n");
            outs.Write(output,0,output.Length);
            outs.WriteByte(Opcodes.EMPTY_TUPLE);
            outs.WriteByte(Opcodes.NEWOBJ);

            // args(8): pyroUri, pyroOneway(hashset), pyroMethods(set), pyroAttrs(set), pyroTimeout, pyroHmacKey, pyroHandshake, pyroMaxRetries
            object[] args = new object[] {
            new PyroURI(proxy.objectid, proxy.hostname, proxy.port),
            proxy.pyroOneway,
            proxy.pyroMethods,
            proxy.pyroAttrs,
            0.0,
            proxy.pyroHmacKey,
            proxy.pyroHandshake,
            0  // maxretries is not yet supported/used by pyrolite
            };
            currentPickler.save(args);
            outs.WriteByte(Opcodes.BUILD);
        }
Example #18
0
	public void testSets() 
	{
		byte[] o;
		Pickler p=new Pickler(false);
		Unpickler up=new Unpickler();

		var intset=new HashSet<int>();
		intset.Add(1);
		intset.Add(2);
		intset.Add(3);
		o=p.dumps(intset);
		HashSet<object> resultset=(HashSet<object>)up.loads(o);
		AssertUtils.AssertEqual(intset, resultset);

		HashSet<string> stringset=new HashSet<string>();
		stringset.Add("A");
		stringset.Add("B");
		stringset.Add("C");
		o=p.dumps(stringset);
		resultset=(HashSet<object>)up.loads(o);
		AssertUtils.AssertEqual(stringset, resultset);
	}
Example #19
0
	public void testMappings() 
	{
		byte[] o;
		Pickler p=new Pickler(false);
		Unpickler pu=new Unpickler();
		var intmap=new Dictionary<int,int>();
		intmap.Add(1, 11);
		intmap.Add(2, 22);
		intmap.Add(3, 33);
		o=p.dumps(intmap);
		Hashtable resultmap=(Hashtable)pu.loads(o);
		AssertUtils.AssertEqual(intmap, resultmap);

		var stringmap=new Dictionary<string,string>();
		stringmap.Add("A", "1");
		stringmap.Add("B", "2");
		stringmap.Add("C", "3");
		o=p.dumps(stringmap);
		resultmap=(Hashtable)pu.loads(o);
		AssertUtils.AssertEqual(stringmap, resultmap);
		
		Hashtable table=new Hashtable();
		table.Add(1,11);
		table.Add(2,22);
		table.Add(3,33);
		o=p.dumps(table);
		resultmap=(Hashtable)pu.loads(o);
		AssertUtils.AssertEqual(table, resultmap);
	}
Example #20
0
	public void testLists()  
	{
		byte[] o;
		Pickler p=new Pickler(false);
		
		IList list=new ArrayList();
		list.Add(1);
		list.Add("abc");
		list.Add(null);
		o=p.dumps(list);
		Assert.AreEqual(B("](K\u0001X\u0003\u0000\u0000\u0000abcNe"), o);
		
		IList<object> ilist=new List<object>();
		ilist.Add(1);
		ilist.Add("abc");
		ilist.Add(null);
		o=p.dumps(ilist);
		Assert.AreEqual(B("](K\u0001X\u0003\u0000\u0000\u0000abcNe"), o);

		Stack<int> stack=new Stack<int>();
		stack.Push(1);
		stack.Push(2);
		stack.Push(3);
		o=p.dumps(stack);
		Assert.AreEqual(B("](K\u0003K\u0002K\u0001e"), o);
		
		var queue=new Queue<int>();
		queue.Enqueue(1);
		queue.Enqueue(2);
		queue.Enqueue(3);
		o=p.dumps(queue);
		Assert.AreEqual(B("](K\u0001K\u0002K\u0003e"), o);
 	}
Example #21
0
        public void testZeroToTwoFiveSix()
        {
            byte[] bytes=new byte[256];
            for(int b=0; b<256; ++b) {
            bytes[b]=(byte)b;
            }
            StringBuilder sb=new StringBuilder();
            for(int i=0; i<256; ++i) {
            sb.Append((char)i);
            }
            string str=sb.ToString();

            Pickler p=new Pickler(false);
            Unpickler u=new Unpickler();

            MemoryStream bos=new MemoryStream(434);
            bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2);
            byte[] data=Encoding.Default.GetBytes("c__builtin__\nbytearray\n");
            bos.Write(data,0,data.Length);
            bos.WriteByte(Opcodes.BINUNICODE);
            bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4);
            byte[] utf8=Encoding.UTF8.GetBytes(str);
            bos.Write(utf8,0,utf8.Length);
            bos.WriteByte(Opcodes.BINUNICODE);
            bos.Write(new byte[] {7,0,0,0},0,4);
            data=Encoding.Default.GetBytes("latin-1");
            bos.Write(data,0,data.Length);
            bos.WriteByte(Opcodes.TUPLE2);
            bos.WriteByte(Opcodes.REDUCE);
            bos.WriteByte(Opcodes.STOP);

            byte[] bytesresult=bos.ToArray();
            byte[] output=p.dumps(bytes);
            Assert.AreEqual(bytesresult, output);
            Assert.AreEqual(bytes, (byte[])u.loads(output));

            bos=new MemoryStream(434);
            bos.WriteByte(Opcodes.PROTO); bos.WriteByte(2);
            bos.WriteByte(Opcodes.BINUNICODE);
            bos.Write(new byte[] {(byte)0x80,0x01,0x00,0x00},0,4);
            utf8=Encoding.UTF8.GetBytes(str);
            bos.Write(utf8,0,utf8.Length);
            bos.WriteByte(Opcodes.STOP);
            bytesresult=bos.ToArray();

            output=p.dumps(str);
            Assert.AreEqual(bytesresult, output);
            Assert.AreEqual(str, u.loads(output));
        }
Example #22
0
        public void testDateTimeStringEscaping()
        {
            DateTime dt=new DateTime(2011, 10, 10, 9, 13, 10, 10);
            Pickler p = new Pickler();
            byte[] pickle = p.dumps(dt);
            Unpickler u = new Unpickler();
            DateTime dt2 = (DateTime) u.loads(pickle);
            Assert.AreEqual(dt, dt2);

            dt = new DateTime(2011, 10, 9, 13, 10, 9, 10);
            dt2 = (DateTime) U("\u0080\u0002cdatetime\ndatetime\nq\u0000U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001\u0085q\u0002Rq\u0003.");	// protocol 2
            Assert.AreEqual(dt, dt2);
            dt2 = (DateTime) U("cdatetime\ndatetime\nq\u0000(U\n\u0007\u00db\n\t\r\n\t\u0000'\u0010q\u0001tq\u0002Rq\u0003.");	// protocol 1
            Assert.AreEqual(dt, dt2);
            dt2 = (DateTime) U("cdatetime\ndatetime\np0\n(S'\\x07\\xdb\\n\\t\\r\\n\\t\\x00'\\x10'\np1\ntp2\nRp3\n.");	// protocol 0
            Assert.AreEqual(dt, dt2);
        }
Example #23
0
	public void testMemoization()  
	{
		byte[] o;
		Pickler p=new Pickler();
		
		string reused = "reused";
		string another = "another";
		IList list=new ArrayList();
		IList sublist = new ArrayList();
		sublist.Add(reused);
		sublist.Add(reused);
		sublist.Add(another);
		list.Add(reused);
		list.Add(reused);
		list.Add(another);
		list.Add(sublist);
		o=p.dumps(list);
		Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01X\x07\x00\x00\x0000anotherq\x02]q\x03(h\x01h\x01h\x0002ee.", S(o));
		
		Unpickler u = new Unpickler();
		ArrayList data = (ArrayList) u.loads(o);
		Assert.AreEqual(4, data.Count);
		string s1 = (string) data[0];
		string s2 = (string) data[1];
		string s3 = (string) data[2];
		data = (ArrayList) data[3];
		string s4 = (string) data[0];
		string s5 = (string) data[1];
		string s6 = (string) data[2];
		Assert.AreEqual("reused", s1);
		Assert.AreEqual("another", s3);
		Assert.AreSame(s1, s2);
		Assert.AreSame(s3, s6);
		Assert.AreSame(s1, s4);
		Assert.AreSame(s1, s5);
	}
Example #24
0
	public void testMemoizationRecursiveMemo()  
	{
		byte[] o;
		Pickler p=new Pickler();
		
		// self-referencing list
		string reused = "reused";
		IList list=new ArrayList();
		list.Add(reused);
		list.Add(reused);
		list.Add(list);
		o=p.dumps(list);
		Assert.AreEqual("\x80\x02]q\x00(X\x06\x00\x00\x0000reusedq\x01h\x01h\x0000e.", S(o));

		Unpickler u = new Unpickler();
		ArrayList data = (ArrayList) u.loads(o);
		Assert.AreEqual(3, data.Count);
		string s1 = (string) data[0];
		string s2 = (string) data[1];
		ArrayList data2 = (ArrayList) data[2];
		Assert.AreEqual("reused", s1);
		Assert.AreSame(s1, s2);
		Assert.AreSame(data, data2);
		Assert.AreSame(data[0], data2[0]);
		
		// self-referencing hashtable
		Hashtable h = new Hashtable();
		h["myself"] = h;
		o=p.dumps(h);
		Assert.AreEqual("\x80\x02}q\x00(X\x06\x00\x00\x0000myselfq\x01h\x0000u.", S(o));
		Hashtable h2 = (Hashtable) u.loads(o);
		Assert.AreEqual(1, h2.Count);
		Assert.AreSame(h2, h2["myself"]);
	}
Example #25
0
	public void testMemoizationRecursiveNoMemo()  
	{
		byte[] o;
		Pickler p=new Pickler(false);
		
		string reused = "reused";
		IList list=new ArrayList();
		list.Add(reused);
		list.Add(reused);
		list.Add(list);
		o=p.dumps(list);
	}
Example #26
0
        private static byte[] GetSerializedMessage(string serializerMode, dynamic message, IFormatter formatter)
        {
            byte[] buffer;

            switch ((SerializedMode)Enum.Parse(typeof(SerializedMode), serializerMode))
            {
                case SerializedMode.None:
                    buffer = message as byte[];
                    break;

                case SerializedMode.String:
                    buffer = SerDe.ToBytes(message as string);
                    break;

                case SerializedMode.Row:
                    var pickler = new Pickler();
                    buffer = pickler.dumps(new ArrayList { message });
                    break;

                default:
                    try
                    {
                        var ms = new MemoryStream();
                        formatter.Serialize(ms, message);
                        buffer = ms.ToArray();
                    }
                    catch (Exception)
                    {
                        logger.LogError("Exception serializing output");
                        logger.LogError("{0} : {1}", message.GetType().Name, message.GetType().FullName);
                        throw;
                    }
                    break;
            }

            return buffer;
        }
Example #27
0
	public void testFailure()
	{
		NotABean notabean=new NotABean();
		notabean.x=42;
		Pickler p=new Pickler(false);
		p.dumps(notabean);
	}
Example #28
0
        public void testUnpicklingPerformance()
        {
            Pickler pickler = new Pickler();

            var myList = new List<string>();
            for (int i = 0; i < 10; i++) {
            myList.Add(i.ToString());
            }

            byte[] bytes = pickler.dumps(myList);

            Unpickler unpickler = new Unpickler();

            DateTime start = DateTime.Now;
            for (int i = 0; i < 1000000; i++) {
            unpickler.loads(bytes);
            }

            Console.WriteLine(DateTime.Now - start);
        }
Example #29
0
		public void pickle(object o, Stream outs, Pickler currentpickler)  {
			CustomClass c=(CustomClass)o;
			currentpickler.save("customclassint="+c.x);		// write a string representation
		}
Example #30
0
        public void TestWorkerWithRowDeserializedModeAndBytesSerializedMode()
        {
            Process worker;
            var CSharpRDD_SocketServer = CreateServer(out worker);

            const int expectedCount = 5;
            using (var serverSocket = CSharpRDD_SocketServer.Accept())
            using (var s = serverSocket.GetStream())
            {
                WritePayloadHeaderToWorker(s);
                byte[] commandWithRowDeserializeMode =
                    SparkContext.BuildCommand(new CSharpWorkerFunc((pid, iter) => iter), SerializedMode.Row);
                SerDe.Write(s, commandWithRowDeserializeMode.Length);
                SerDe.Write(s, commandWithRowDeserializeMode);

                new StructTypePickler().Register();
                new RowPickler().Register();
                Pickler pickler = new Pickler();

                for (int i = 0; i < expectedCount; i++)
                {
                    byte[] pickleBytes = pickler.dumps(new[] { RowHelper.BuildRowForBasicSchema(i) });
                    SerDe.Write(s, pickleBytes.Length);
                    SerDe.Write(s, pickleBytes);
                }

                SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
                SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
                s.Flush();

                int count = 0;
                var formatter = new BinaryFormatter();
                foreach (var bytes in ReadWorker(s))
                {
                    var ms = new MemoryStream(bytes);
                    var rows = new ArrayList { formatter.Deserialize(ms) }.Cast<Row>().ToArray();
                    Assert.AreEqual(1, rows.Count());
                    Assert.AreEqual(count, rows[0].Get("age"));
                    count++;

                }

                Assert.AreEqual(expectedCount, count);
            }

            AssertWorker(worker);
            CSharpRDD_SocketServer.Close();
        }
Example #31
0
	public void testCustomPickler() 
	{
		Pickler.registerCustomPickler(typeof(CustomClass), new CustomClassPickler());
		CustomClass c=new CustomClass();
		Pickler p=new Pickler(false);
		byte[] ser = p.dumps(c);
		
		Unpickler u = new Unpickler();
		string x = (string) u.loads(ser);
		Assert.AreEqual("customclassint=42", x);
	}