Esempio n. 1
0
        static void Main(string[] args)
        {
            ObjectEntity objectEntity = new ObjectEntity();

            objectEntity.NullableIntField = 10;
            objectEntity.StringField      = "teststring";
            objectEntity.ListField        = new List <SubObjectEntity>();
            objectEntity.ListField.Add(new SubObjectEntity {
                IntField = 1, DoubleField = 1.1
            });

            Schema schema = Schema.Parse(SchemaClass.SCHEMA_OBJECT_ENTITY);
            SpecificDatumWriter <ObjectEntity> w = new SpecificDatumWriter <ObjectEntity>(schema);

            byte[] finalbytes = null;
            using (MemoryStream memStream = new MemoryStream())
            {
                w.Write(objectEntity, new Avro.IO.BinaryEncoder(memStream));
                memStream.Seek(0, SeekOrigin.Begin);
                finalbytes = memStream.ToArray();
            }

            EventData      eventToSend    = new EventData(finalbytes);
            EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(
                "<eventhubconnection>",
                "<eventhubname>"
                );

            eventHubClient.SendAsync(eventToSend).Wait();
        }
Esempio n. 2
0
        private static void checkAlternateSerializers <T>(byte[] expected, T value, Schema ws)
        {
            var ms     = new MemoryStream();
            var writer = new SpecificDatumWriter <T>(ws);
            var e      = new BinaryEncoder(ms);

            writer.Write(value, e);
            var output = ms.ToArray();

            Assert.AreEqual(expected.Length, output.Length);
            Assert.True(expected.SequenceEqual(output));
        }
Esempio n. 3
0
        private sbyte[] Write <T>(T message, SpecificDatumWriter <T> writer)
        {
            var     ms = new MemoryStream();
            Encoder e  = new BinaryEncoder(ms);

            writer.Write(message, e);
            ms.Flush();
            ms.Position = 0;
            var b = ms.ToArray();

            return((sbyte[])(object)b);
        }
Esempio n. 4
0
        private byte[] SpecificRecordsToAvro <T>(T record) where T : ISpecificRecord
        {
            using (MemoryStream outputStream = new MemoryStream())
            {
                SpecificDatumWriter <T> writer  = new SpecificDatumWriter <T>(record.Schema);
                BinaryEncoder           encoder = new BinaryEncoder(outputStream);

                for (int i = 0; i < _numberOfRecordsInAvro; i++)
                {
                    writer.Write(record, encoder);
                }

                encoder.Flush();

                return(outputStream.ToArray());
            }
        }
        protected override byte[] DoEncode(string topic, object obj)
        {
            if (!(obj is ISpecificRecord))
            {
                throw new InvalidDataException(string.Format("{0} should be instance of ISpecificRecord to user Avro", typeof(object)));                
            }

            Schema schema = ((ISpecificRecord)obj).Schema;

            int schemaId = schemaCache.RegisterSchema(schema.ToString(), topic + "-value");
            if (!IsValidSchemaId(schemaId))
            {
                throw new InvalidDataException(string.Format("Avro schema of {0} is incompaible with latest schema", typeof(object)));                
            }


            SpecificDatumWriter<object> w = new SpecificDatumWriter<object>(schema);

            using (MemoryStream stream = new MemoryStream())
            {
                // same with io.confluent.kafka used by Java
                // prepend body with magic byte and schemaId
                stream.WriteByte(MAGIC_BYTE);
                byte[] schemaIdBytes = BitConverter.GetBytes(schemaId);
                if (BitConverter.IsLittleEndian)
                {
                    Array.Reverse(schemaIdBytes);
                }
                foreach (byte b in schemaIdBytes)
                {
                    stream.WriteByte(b);
                }

                w.Write(obj, new BinaryEncoder(stream));

                stream.Seek(0, SeekOrigin.Begin);
                byte[] result = new byte[stream.Length];
                stream.Read(result, 0, result.Length);
                return result;
            }
        }
Esempio n. 6
0
 public byte[] Serialize(object obj)
 {
     try
     {
         ISpecificRecord record = (ISpecificRecord)obj;
         writer = new SpecificDatumWriter <ISpecificRecord>(record.Schema);
         using (MemoryStream ms = new MemoryStream())
         {
             int function = (int)record.Get(0);
             ms.Write(BitConverter.GetBytes(function), 0, 4);
             BinaryEncoder encoder = new BinaryEncoder(ms);
             writer.Write(record, encoder);
             return(ms.ToArray());
         }
     }
     catch (Exception e)
     {
         Console.WriteLine(e.Message);
     }
     return(null);
 }
Esempio n. 7
0
        public void CreateTestData()
        {
            RuntimeTypeModel.Default.Add(typeof(DateTimeOffset), false).SetSurrogate(typeof(DateTimeOffsetSurrogate));

            JsonConvert.DefaultSettings = () => new JsonSerializerSettings
            {
                ContractResolver = new CamelCasePropertyNamesContractResolver()
            };

            var @event = new TestEvent(Guid.NewGuid(), RandomString(5), random.NextDouble(), random.NextDouble(), random.NextDouble(), DateTimeOffset.Now);

            _event = @event;

            using (var stream = new MemoryStream())
            {
                Serializer.Serialize(stream, @event);
                ProtoSerializedData = stream.ToArray();
            }

            var jilJsonString = JSON.Serialize(@event);

            JilSerializedData = Encoding.UTF8.GetBytes(jilJsonString);
            var protoMessage = new Testevents.Protobuf.TestEvent
            {
                Id        = ByteString.CopyFrom(@event.Id.ToByteArray()),
                Name      = @event.Name,
                X         = @event.X,
                Y         = @event.Y,
                Z         = @event.Z,
                TimeStamp = Timestamp.FromDateTimeOffset(@event.TimeStamp),
            };

            GProtoSerialziedData = SerializeGProto(protoMessage);

            var msgPackResult = MessagePackSerializer.Serialize(@event);

            MsgPackData = msgPackResult;

            var msgPackKeyString = new TestEventMessagePackNoKeys
            {
                Id        = @event.Id,
                Name      = @event.Name,
                X         = @event.X,
                Y         = @event.Y,
                Z         = @event.Z,
                TimeStamp = @event.TimeStamp
            };

            _event2 = msgPackKeyString;
            var msgPackStringKeysResult = MessagePackSerializer.Serialize(msgPackKeyString);

            MsgPackDataIntKeys = msgPackStringKeysResult;

            var newtonJsonString = JsonConvert.SerializeObject(@event);

            NewtonsoftData = Encoding.UTF8.GetBytes(newtonJsonString);

            var avroEvent = new Avro.TestEvent
            {
                id        = @event.Id.ToByteArray(),
                name      = @event.Name,
                timeStamp = @event.TimeStamp.ToUnixTimeMilliseconds(),
                x         = @event.X,
                y         = @event.Y,
                z         = @event.Z
            };

            using (var stream = new MemoryStream())
            {
                avroSerializer.Write(avroEvent, new BinaryEncoder(stream));
                AvroData = stream.ToArray();
            }

            Utf8JsonData = Utf8Json.JsonSerializer.Serialize(@event, StandardResolver.CamelCase);
            spreadsData  = Spreads.Serialization.Utf8Json.JsonSerializer.Serialize(@event,
                                                                                   Spreads.Serialization.Utf8Json.Resolvers.StandardResolver.CamelCase);
            zeroFormatterData = ZeroFormatterSerializer.Serialize(@event);
            SystemTextData    = System.Text.Json.Serialization.JsonSerializer.ToUtf8Bytes(@event);

            var output    = new OutputBuffer();
            var writer    = new CompactBinaryWriter <OutputBuffer>(output);
            var bondEvent = new Bond.TestEvent
            {
                id   = @event.Id.ToByteArray(),
                name = @event.Name,
                x    = @event.X,
                y    = @event.Y,
                z    = @event.Z,
                date = @event.TimeStamp.ToUnixTimeMilliseconds()
            };

            Bond.Serialize.To(writer, bondEvent);
            bondData = output.Data.ToArray();

            /*Console.WriteLine($"Size of jil object {JilSerializedData.First().Length}");
             * Console.WriteLine($"Size of protobuf-net object {ProtoSerializedData.First().Length}");
             * Console.WriteLine($"Size of gproto object {GProtoSerialziedData.First().Length}");
             * Console.WriteLine($"Size of msgPack object {MsgPackData.First().Length}");
             * Console.WriteLine($"Size of msgPack object with int keys {MsgPackDataIntKeys.First().Length}");
             * Console.WriteLine($"Size of avro data {AvroData.First().Length}");
             * Console.WriteLine($"Size of newtonsoft object {NewtonsoftData.First().Length}");
             * Console.WriteLine($"Size of utf8json object {Utf8JsonData.First().Length}");
             * Console.WriteLine($"Size of zeroformatter object {zeroFormatterData.First().Length}");*/
        }
Esempio n. 8
0
        public static void test ()
        {
            //Schema schema = Schema.Parse ("schema json");

            SpecificDatumWriter<User> w = new SpecificDatumWriter<User> (User._SCHEMA);
            User input = new User ();
            input.name = "mm";
            input.favorite_color = "red";
            input.favorite_number = 11;

            MemoryStream stream = new MemoryStream ();
            w.Write (input, new BinaryEncoder (stream));

            stream.Seek (0, SeekOrigin.Begin);

            SpecificDatumReader<User> r = new SpecificDatumReader<User> (User._SCHEMA, User._SCHEMA);
            User output = r.Read (null, new BinaryDecoder (stream));
            Console.WriteLine (output.name);
        }