public StreamWriter(Stream stream, bool leaveOpen, IAvroSerializer <T> serializer, Codec codec)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (serializer == null)
{
throw new ArgumentNullException("serializer");
}
if (codec == null)
{
throw new ArgumentNullException("codec");
}
this.codec = codec;
this.serializer = serializer;
this.resultStream = stream;
this.encoder = new BinaryEncoder(this.resultStream, leaveOpen);
this.isHeaderWritten = false;
this.locker = new object();
var syncMarker = new byte[16];
new Random().NextBytes(syncMarker);
this.header = new ObjectContainerHeader(syncMarker)
{
CodecName = this.codec.Name, Schema = this.serializer.WriterSchema.ToString()
};
}
static Serializer()
{
ser = AvroSerializer.Create <T>(new AvroSerializerSettings()
{
UseCache = true
});
}
/// <summary>
/// Generate and store the metadata necessary to serialze and deserialize a specific message type.
/// </summary>
/// <typeparam name="TMessage">The class type of the message being registered.</typeparam>
internal void Register <TMessage>()
{
Logr.Log(Level.Info, "Registering message type: {0} {1}",
typeof(TMessage).FullName, typeof(TMessage).Name);
IAvroSerializer <TMessage> messageSerializer = AvroSerializer.Create <TMessage>();
Serialize serialize = (MemoryStream stream, object message) =>
{
messageSerializer.Serialize(stream, (TMessage)message);
};
serializeMap.Add(typeof(TMessage).Name, serialize);
Deserialize deserialize = (MemoryStream stream, object observer, long sequence) =>
{
TMessage message = messageSerializer.Deserialize(stream);
var msgObserver = observer as IObserver <IMessageInstance <TMessage> >;
if (msgObserver != null)
{
msgObserver.OnNext(new MessageInstance <TMessage>(sequence, message));
}
else
{
Logr.Log(Level.Warning, "Unhandled message received: {0}", message);
}
};
deserializeMap.Add(typeof(TMessage).Name, deserialize);
}
private IEnumerable <object> ReadObjects <T>(StreamReader sr, IAvroSerializer <T> avroSerializer, Func <object, bool?> filterFunc = null)
{
while (true)
{
object obj = null;
try
{
obj = avroSerializer.Deserialize(sr.BaseStream);
if (IsDynamicType)
{
obj = new ChoDynamicObject(obj as Dictionary <string, object>);
}
}
catch (System.OverflowException)
{
break;
}
catch (SerializationException sEx)
{
if (sEx.Message.StartsWith("Invalid integer value in the input stream"))
{
break;
}
throw;
}
yield return(obj);
}
}
/// <summary>
/// Converts a byte array to an object using Avro deserialization.
/// </summary>
/// <param name="data">The byte array to deserialize</param>
/// <returns>The deserialized object</returns>
public static T AvroDeserialize <T>(byte[] data)
{
IAvroSerializer <T> deserializer = AvroSerializer.Create <T>();
using (MemoryStream stream = new MemoryStream(data))
{
return(deserializer.Deserialize(stream));
}
}
public void JsonSchemaBuilder_ClassOfGuidWriterSchema()
{
var settings = new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(false)
};
IAvroSerializer <ClassOfGuid> serializer = AvroSerializer.Create <ClassOfGuid>(settings);
string writerSchema = serializer.WriterSchema.ToString();
AvroSerializer.CreateDeserializerOnly <ClassOfGuid>(writerSchema, settings);
}
/// <summary>
/// Convert an object to byte array using Avro serialization
/// </summary>
/// <param name="obj">The object to serialize</param>
/// <returns>The serialized object in a byte array</returns>
public static byte[] AvroSerialize <T>(T obj)
{
IAvroSerializer <T> serializer = AvroSerializer.Create <T>();
using (MemoryStream stream = new MemoryStream())
{
serializer.Serialize(stream, obj);
return(stream.GetBuffer());
}
}
public AvroTestKafkaModelDeserializer()
{
var schemaString = "{\"type\": \"record\", " +
"\"name\": \"kevent\"," +
"\"fields\": [" +
"{\"name\": \"timestamp\", \"type\": \"long\"}," +
"{\"name\": \"payload\", \"type\": \"bytes\"}" +
"]}";
avroSerializer = AvroSerializer.CreateGeneric(schemaString);
}
public AvroBufferWriterBlock(IAvroSerializer <T> serializer, Codec codec)
{
if (serializer == null)
{
throw new ArgumentNullException("serializer");
}
if (codec == null)
{
throw new ArgumentNullException("codec");
}
this.serializer = serializer;
this.buffer = new MemoryStream();
this.compressed = codec.GetCompressedStreamOver(this.buffer);
this.encoder = new BufferedBinaryEncoder(this.compressed);
this.objectCount = 0;
}
/// <summary>
/// Initializes a new instance of the <see cref="StreamReader{T}"/> class for Avro records.
/// </summary>
/// <param name="readerSchema">The reader schema.</param>
/// <param name="stream">The input stream.</param>
/// <param name="leaveOpen">If set to <c>true</c> leaves the input stream open.</param>
/// <param name="codecFactory">The codec factory.</param>
public StreamReader(string readerSchema, Stream stream, bool leaveOpen, CodecFactory codecFactory)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (codecFactory == null)
{
throw new ArgumentNullException("codecFactory");
}
this.stream = stream;
this.decoder = new BinaryDecoder(stream, leaveOpen);
this.header = ObjectContainerHeader.Read(this.decoder);
this.codec = codecFactory.Create(this.header.CodecName);
this.serializer = (IAvroSerializer <T>)AvroSerializer.CreateGenericDeserializerOnly(this.header.Schema, readerSchema ?? this.header.Schema);
}
public void GenerateObject()
{
var config = new MapperConfiguration(cfg =>
{
cfg.AddConditionalObjectMapper().Where((s, d) => d.Name.Contains(s.Name));
});
this.sampleStringObject = (TBase)factories[typeof(TBase)];
var mapper = config.CreateMapper();
this.sampleStringObjectSerializable = mapper.Map <TPoco>(this.sampleStringObject);
this.dataContractObject = mapper.Map <TContract>(this.sampleStringObject);
this.bondObject = mapper.Map <TBond>(this.sampleStringObject);
this.jsonObject = mapper.Map <TBase>(this.sampleStringObject);
this.thriftObject = mapper.Map <TThrift>(this.sampleStringObject);
this.jsonSerializer = new JsonSerializer()
{
NullValueHandling = NullValueHandling.Ignore
};
this.dataContractSerializer = new DataContractSerializer(typeof(TContract));
this.javascriptDataContractSerializer = new DataContractJsonSerializer(typeof(TContract));
this.avroSerializer = Microsoft.Hadoop.Avro.AvroSerializer.Create <TContract>();
this.messagePackSerializer = MessagePackSerializer.Get <TBase>();
this.compactBondSerializer = new Serializer <CompactBinaryWriter <OutputBuffer> >(typeof(TBond));
this.simpleBondSerializer = new Serializer <SimpleBinaryWriter <OutputBuffer> >(typeof(TBond));
this.javascriptSerializer = new JavaScriptSerializer();
// can't use automapper here because I have null
// this.protoObject = mapper.Map<SmallObjectWithStringsProtobuf>(this.sampleStringObject);
Func <object, object> map;
if (manualMapFromTypeToType.TryGetValue(Tuple.Create <Type, Type>(typeof(TBase), typeof(TProtobuf)), out map))
{
this.protoObject = (IMessage)map(this.sampleStringObject);
}
else
{
mapper.Map <TProtobuf>(this.sampleStringObject);
}
}
internal void Dispose <T>()
{
if (_sw != null)
{
RaiseEndWrite(_sw);
}
else if (_avroWriter != null)
{
RaiseEndWrite(_avroWriter);
}
if (Configuration.UseAvroSerializer)
{
if (IsDynamicType)
{
IAvroSerializer <Dictionary <string, object> > avroSerializer = _avroSerializer as IAvroSerializer <Dictionary <string, object> >;
}
else
{
IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>;
}
}
else
{
if (IsDynamicType)
{
SequentialWriter <Dictionary <string, object> > avroWriter = _avroWriter as SequentialWriter <Dictionary <string, object> >;
if (avroWriter != null)
{
avroWriter.Dispose();
}
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
if (avroWriter != null)
{
avroWriter.Dispose();
}
}
}
}
public override void Output(IRow row, IUnstructuredWriter output)
{
// First Row
if (this.writer == null)
{
//this.writer = new SequentialWriter<object>(AvroContainer.Create<AnonType_WeatherDataSet>(this.avroSchema, output.BaseStream, Codec.Deflate), 24);
this.serializer = AvroSerializer.Create <AnonType_WeatherDataSet>();
}
AnonType_WeatherDataSet data = new AnonType_WeatherDataSet(
row.Get <string>(row.Schema[0].Name),
row.Get <int?>(row.Schema[1].Name),
row.Get <string>(row.Schema[2].Name),
(double?)row.Get <decimal?>(row.Schema[3].Name),
row.Get <string>(row.Schema[4].Name),
row.Get <string>(row.Schema[5].Name),
row.Get <string>(row.Schema[6].Name),
row.Get <int?>(row.Schema[7].Name)
);
serializer.Serialize(output.BaseStream, data);
}
public override void Output(IRow row, IUnstructuredWriter output)
{
// First Row
if (this.writer == null)
{
this.writer = new SequentialWriter <object>(AvroContainer.CreateGenericWriter(this.avroSchema, output.BaseStream, Codec.Deflate), 24);
this.serializer = AvroSerializer.CreateGeneric(this.avroSchema);
}
dynamic data = new AvroRecord(this.serializer.WriterSchema);
data.station = row.Get <string>(row.Schema[0].Name);
data.datekey = row.Get <int?>(row.Schema[1].Name);
data.element = row.Get <string>(row.Schema[2].Name);
data.value = (double?)row.Get <decimal?>(row.Schema[3].Name);
data.mflag = row.Get <string>(row.Schema[4].Name);
data.qflag = row.Get <string>(row.Schema[5].Name);
data.sflag = row.Get <string>(row.Schema[6].Name);
data.timekey = row.Get <int?>(row.Schema[7].Name);
writer.Write(data);
}
/// <summary>
/// Initializes a new instance of the <see cref="StreamReader{T}"/> class for static types.
/// </summary>
/// <param name="stream">The stream.</param>
/// <param name="leaveOpen">If set to <c>true</c> leaves the input stream open.</param>
/// <param name="settings">The settings.</param>
/// <param name="codecFactory">The codec factory.</param>
public StreamReader(Stream stream, bool leaveOpen, AvroSerializerSettings settings, CodecFactory codecFactory)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (settings == null)
{
throw new ArgumentNullException("settings");
}
if (codecFactory == null)
{
throw new ArgumentNullException("codecFactory");
}
this.stream = stream;
this.decoder = new BinaryDecoder(stream, leaveOpen);
this.header = ObjectContainerHeader.Read(this.decoder);
this.codec = codecFactory.Create(this.header.CodecName);
this.serializer = AvroSerializer.CreateDeserializerOnly <T>(this.header.Schema, settings);
}
public AvroBlobAppenderWriter(CloudBlockBlob blockBlob, bool leaveOpen, IAvroSerializer <T> serializer, Codec codec)
{
if (blockBlob == null)
{
throw new ArgumentNullException("blockBlob");
}
if (serializer == null)
{
throw new ArgumentNullException("serializer");
}
if (codec == null)
{
throw new ArgumentNullException("codec");
}
this.codec = codec;
this.serializer = serializer;
this.isHeaderWritten = false;
this.locker = new object();
this.blockBlob = blockBlob;
this.SetupObjectContainerHeader();
}
public AvroBufferReaderBlock(IAvroSerializer <T> serializer, Codec codec, byte[] rawBlock, int objectCount)
{
if (serializer == null)
{
throw new ArgumentNullException("serializer");
}
if (codec == null)
{
throw new ArgumentNullException("codec");
}
if (rawBlock == null)
{
throw new ArgumentNullException("rawBlock");
}
if (objectCount < 0)
{
throw new ArgumentOutOfRangeException("objectCount");
}
this.serializer = serializer;
this.codec = codec;
this.rawBlock = rawBlock;
this.objectCount = objectCount;
}
public virtual ChoAvroReader <T> WithAvroSerializer(IAvroSerializer <T> avroSerializer)
{
AvroSerializer = avroSerializer;
Configuration.UseAvroSerializer = true;
return(this);
}
public AvroSerializer()
{
_serializer = Microsoft.Hadoop.Avro.AvroSerializer.Create<object>();
}
internal MSHadoopAvroSerializer()
: base(new Catalog())
{
serializer = AvroSerializer.Create <Catalog>();
}
public Serializer()
{
_serializer = AvroSerializer.Create <T>();
}
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
Configuration.Init();
if (records == null)
{
yield break;
}
var recEnum = records.GetEnumerator();
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
DiscoverKnownTypes(notNullRecord);
StreamWriter sw = null;
if (writer is Lazy <StreamWriter> )
{
var lsw = writer as Lazy <StreamWriter>;
ChoGuard.ArgumentNotNull(lsw, "StreamWriter");
_sw = sw = lsw.Value;
if (!Configuration.UseAvroSerializer)
{
if (_avroWriter == null)
{
_avroWriter = new SequentialWriter <T>(CreateAvroWriter <T>(sw), Configuration.SyncNumberOfObjects);
}
}
else
{
_avroSerializer = CreateAvroSerializer <T>();
}
}
else
{
_avroWriter = writer as IAvroWriter <T>;
if (_avroWriter == null)
{
throw new ChoParserException("Missing valid writer object passed.");
}
}
if (!BeginWrite.Value)
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
object recOutput = null;
try
{
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recOutput = record;
if (record != null)
{
if (predicate == null || predicate(record))
{
if (!RaiseBeforeRecordWrite(record, _index, ref recOutput))
{
yield break;
}
if (recOutput == null)
{
continue;
}
else if (!(recOutput is T))
{
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations);
}
if (recOutput is T)
{
if (recOutput is ChoDynamicObject)
{
recOutput = new Dictionary <string, object>((ChoDynamicObject)recOutput);
}
if (_sw != null)
{
if (Configuration.UseAvroSerializer)
{
IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>;
avroSerializer.Serialize(sw.BaseStream, (T)(recOutput as object));
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
if (!RaiseAfterRecordWrite(record, _index, recOutput))
{
yield break;
}
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recOutput, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return((T)recOutput);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
public AvroDeserializer()
{
avroSerializer = AvroSerializer.Create <T>();
}
public AvroSerializer()
{
_serializer = Microsoft.Hadoop.Avro.AvroSerializer.Create <object>();
}
public SampleAvroSerializer()
{
avroSerializer = AvroSerializer.Create <T>();
}
