public override string Serialize(object serializable)
{
using (var ms = new MemoryStream())
{
_serializer.Serialize(ms, serializable);
ms.Flush();
ms.Position = 0;
return(Convert.ToBase64String(ms.ToArray()));
}
}
/// <summary>
/// Generate and store the metadata necessary to serialze and deserialize a specific message type.
/// </summary>
/// <typeparam name="TMessage">The class type of the message being registered.</typeparam>
internal void Register <TMessage>()
{
Logr.Log(Level.Info, "Registering message type: {0} {1}",
typeof(TMessage).FullName, typeof(TMessage).Name);
IAvroSerializer <TMessage> messageSerializer = AvroSerializer.Create <TMessage>();
Serialize serialize = (MemoryStream stream, object message) =>
{
messageSerializer.Serialize(stream, (TMessage)message);
};
serializeMap.Add(typeof(TMessage).Name, serialize);
Deserialize deserialize = (MemoryStream stream, object observer, long sequence) =>
{
TMessage message = messageSerializer.Deserialize(stream);
var msgObserver = observer as IObserver <IMessageInstance <TMessage> >;
if (msgObserver != null)
{
msgObserver.OnNext(new MessageInstance <TMessage>(sequence, message));
}
else
{
Logr.Log(Level.Warning, "Unhandled message received: {0}", message);
}
};
deserializeMap.Add(typeof(TMessage).Name, deserialize);
}
/// <summary>
/// Serialize the input message and return a byte array.
/// </summary>
/// <param name="message">An object reference to the messeage to be serialized</param>
/// <param name="sequence">
/// A long which cotains the higher level protocols sequence number for the message.</param>
/// <returns>A byte array containing the serialized header and message.</returns>
public byte[] Write(object message, long sequence)
{
string name = message.GetType().Name;
Logr.Log(Level.Info, "Serializing message: {0}", name);
try
{
using (MemoryStream stream = new MemoryStream())
{
Header header = new Header(sequence, name);
headerSerializer.Serialize(stream, header);
Serialize serialize;
if (serializeMap.TryGetValue(name, out serialize))
{
serialize(stream, message);
}
else
{
throw new SeializationException(
"Request to serialize unknown message type: " + name);
}
return(stream.GetBuffer());
}
}
catch (Exception e)
{
Logr.Log(Level.Error, "Failure writing message.", e);
throw e;
}
}
public static byte[] ToBytes(T obj)
{
using (var buffer = new MemoryStream())
{
ser.Serialize(buffer, obj);
return(buffer.ToArray());
}
}
public byte[] Serialize(T objectToSerialize)
{
using (var memStream = new MemoryStream())
{
avroSerializer.Serialize(memStream, objectToSerialize);
return(memStream.ToArray());
}
}
public byte[] Serialize(T data)
{
using (var buffer = new MemoryStream())
{
_serializer.Serialize(buffer, data);
return(buffer.ToArray());
}
}
/// <summary>
/// Convert an object to byte array using Avro serialization
/// </summary>
/// <param name="obj">The object to serialize</param>
/// <returns>The serialized object in a byte array</returns>
public static byte[] AvroSerialize <T>(T obj)
{
IAvroSerializer <T> serializer = AvroSerializer.Create <T>();
using (MemoryStream stream = new MemoryStream())
{
serializer.Serialize(stream, obj);
return(stream.GetBuffer());
}
}
/// <summary>
/// Write SecurityToken objects to SecurityTokenFile using IAvroSerializer.
/// </summary>
public void WriteTokensToFile()
{
Logger.Log(Level.Verbose, "Write {0} tokens to file: {1}.", _tokens.Count, _securityTokensFile);
if (_tokens.Count > 0)
{
using (var stream = File.OpenWrite(_securityTokensFile))
{
foreach (var token in _tokens)
{
Logger.Log(Level.Verbose, "Write token {0} to file.", token.kind);
_avroSerializer.Serialize(stream, token);
}
}
}
}
public Task <DeliveryReport> SendAsync(K key, V value, int partition)
{
using (var keyStream = new MemoryStream())
using (var keyWriter = new BinaryWriter(keyStream))
using (var valueStream = new MemoryStream())
using (var valueWriter = new BinaryWriter(valueStream))
{
keyWriter.Write((byte)0);
keyWriter.Write(_keySchemaBytes);
valueWriter.Write((byte)0);
valueWriter.Write(_valueSchemaBytes);
_keySerializer.Serialize(keyStream, key);
_valueSerializer.Serialize(valueStream, value);
return(_topic.Produce(keyStream.ToArray(), valueStream.ToArray(), partition));
}
}
public override void Output(IRow row, IUnstructuredWriter output)
{
// First Row
if (this.writer == null)
{
//this.writer = new SequentialWriter<object>(AvroContainer.Create<AnonType_WeatherDataSet>(this.avroSchema, output.BaseStream, Codec.Deflate), 24);
this.serializer = AvroSerializer.Create <AnonType_WeatherDataSet>();
}
AnonType_WeatherDataSet data = new AnonType_WeatherDataSet(
row.Get <string>(row.Schema[0].Name),
row.Get <int?>(row.Schema[1].Name),
row.Get <string>(row.Schema[2].Name),
(double?)row.Get <decimal?>(row.Schema[3].Name),
row.Get <string>(row.Schema[4].Name),
row.Get <string>(row.Schema[5].Name),
row.Get <string>(row.Schema[6].Name),
row.Get <int?>(row.Schema[7].Name)
);
serializer.Serialize(output.BaseStream, data);
}
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
Configuration.Init();
if (records == null)
{
yield break;
}
var recEnum = records.GetEnumerator();
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
DiscoverKnownTypes(notNullRecord);
StreamWriter sw = null;
if (writer is Lazy <StreamWriter> )
{
var lsw = writer as Lazy <StreamWriter>;
ChoGuard.ArgumentNotNull(lsw, "StreamWriter");
_sw = sw = lsw.Value;
if (!Configuration.UseAvroSerializer)
{
if (_avroWriter == null)
{
_avroWriter = new SequentialWriter <T>(CreateAvroWriter <T>(sw), Configuration.SyncNumberOfObjects);
}
}
else
{
_avroSerializer = CreateAvroSerializer <T>();
}
}
else
{
_avroWriter = writer as IAvroWriter <T>;
if (_avroWriter == null)
{
throw new ChoParserException("Missing valid writer object passed.");
}
}
if (!BeginWrite.Value)
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
object recOutput = null;
try
{
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recOutput = record;
if (record != null)
{
if (predicate == null || predicate(record))
{
if (!RaiseBeforeRecordWrite(record, _index, ref recOutput))
{
yield break;
}
if (recOutput == null)
{
continue;
}
else if (!(recOutput is T))
{
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations);
}
if (recOutput is T)
{
if (recOutput is ChoDynamicObject)
{
recOutput = new Dictionary <string, object>((ChoDynamicObject)recOutput);
}
if (_sw != null)
{
if (Configuration.UseAvroSerializer)
{
IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>;
avroSerializer.Serialize(sw.BaseStream, (T)(recOutput as object));
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
if (!RaiseAfterRecordWrite(record, _index, recOutput))
{
yield break;
}
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recOutput, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return((T)recOutput);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
public override void Serialize(Stream stream, Catalog data)
{
serializer.Serialize(stream, data);
}
private void Serialize(Stream stream, object obj)
{
_serializer.Serialize(stream, (ObjT)obj);
}