public void Write(T record)
{
if (record is DataTable)
{
Write(record as DataTable);
return;
}
else if (record is IDataReader)
{
Write(record as IDataReader);
return;
}
if (_streamWriter != null)
{
_avroWriter = Create(_streamWriter.Value);
}
_writer.Writer = this;
_writer.TraceSwitch = TraceSwitch;
if (record is ArrayList)
{
_writer.WriteTo <T>(_avroWriter, ((IEnumerable)record).AsTypedEnumerable <object>()).Loop();
}
else if (record != null && !(/*!record.GetType().IsDynamicType() && record is IDictionary*/ record.GetType() == typeof(ExpandoObject) || typeof(IDynamicMetaObjectProvider).IsAssignableFrom(record.GetType()) || record.GetType() == typeof(object) || record.GetType().IsAnonymousType()) &&
(typeof(IDictionary).IsAssignableFrom(record.GetType()) || (record.GetType().IsGenericType&& record.GetType().GetGenericTypeDefinition() == typeof(IDictionary <,>))))
{
_writer.WriteTo <T>(_avroWriter, ((IEnumerable)record).AsTypedEnumerable <object>()).Loop();
}
else
{
_writer.WriteTo <T>(_avroWriter, new object[] { record }).Loop();
}
}
protected ChoAvroWriter(IAvroWriter <Dictionary <string, object> > avroWriter, ChoAvroRecordConfiguration configuration = null)
{
ChoGuard.ArgumentNotNull(avroWriter, "AvroWriter");
Configuration = configuration;
Init();
_avroWriter = avroWriter;
}
public ChoAvroWriter(IAvroWriter <T> avroWriter, ChoAvroRecordConfiguration configuration = null)
{
ChoGuard.ArgumentNotNull(avroWriter, "AvroWriter");
Configuration = configuration;
Init();
_avroWriter = avroWriter;
}
public void Write(IEnumerable <T> records)
{
if (_streamWriter != null)
{
_avroWriter = Create(_streamWriter.Value);
}
_writer.Writer = this;
_writer.TraceSwitch = TraceSwitch;
_writer.WriteTo <T>(_avroWriter, records.OfType <object>()).Loop();
}
public override void WriteAvroFile(Stream stream)
{
using (IAvroWriter <object> streamWriter = AvroContainer.CreateGenericWriter(this.GetSchema().ToString(), stream, this.Codec))
{
using (var sequentialWriter = new SequentialWriter <object>(streamWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize))
{
foreach (LargeClass record in this.Expected)
{
sequentialWriter.Write(record.ToAvroRecord(this.GetSchema()));
}
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="SequentialWriter{T}"/> class.
/// </summary>
/// <param name="writer">The writer.</param>
/// <param name="syncNumberOfObjects">The sync number of objects.</param>
/// <exception cref="System.ArgumentNullException">Thrown when <paramref name="writer"/> is null.</exception>
public SequentialWriter(IAvroWriter <T> writer, int syncNumberOfObjects)
{
if (writer == null)
{
throw new ArgumentNullException("writer");
}
if (syncNumberOfObjects <= 0)
{
throw new ArgumentOutOfRangeException("syncNumberOfObjects");
}
this.writer = writer;
this.metadata = new Dictionary <string, byte[]>();
this.metadataWritten = false;
this.current = writer.CreateBlock();
this.syncNumberOfObjects = syncNumberOfObjects;
}
public override void WriteAvroFile(Stream stream)
{
IAvroWriter <LargeClass> avroWriter = AvroContainer.CreateWriter <LargeClass>(
stream,
new AvroSerializerSettings {
UsePosixTime = true
},
this.Codec);
using (var writer = new SequentialWriter <LargeClass>(avroWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize))
{
foreach (LargeClass record in this.Expected)
{
writer.Write(record);
}
}
}
public void TestReadFeed()
{
MemoryStream ms = new MemoryStream();
IAvroWriter <Product[]> writer = null;
try
{
writer = AvroContainer.CreateWriter <Product[]>(ms, true, new AvroSerializerSettings(), Codec.Null);
using (var seqWriter = new SequentialWriter <Product[]>(writer, 24))
{
seqWriter.Write(new[] { product1, product2, product3 });
}
}
finally
{
if (writer != null)
{
writer.Dispose();
}
}
ms.Seek(0, SeekOrigin.Begin);
var entries = new List <ODataEntry>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataFeedReader();
while (reader.Read())
{
if (reader.State == ODataReaderState.EntryEnd)
{
entries.Add((ODataEntry)reader.Item);
}
}
}
Assert.AreEqual(3, entries.Count);
Assert.IsTrue(TestHelper.EntryEqual(entry1, entries[0]));
Assert.IsTrue(TestHelper.EntryEqual(entry2, entries[1]));
Assert.IsTrue(TestHelper.EntryEqual(entry3, entries[2]));
}
public void TestReadEntry()
{
MemoryStream ms = new MemoryStream();
IAvroWriter <Product> writer = null;
try
{
writer = AvroContainer.CreateWriter <Product>(ms, true, new AvroSerializerSettings(), Codec.Null);
using (var seqWriter = new SequentialWriter <Product>(writer, 24))
{
seqWriter.Write(product0);
}
}
finally
{
if (writer != null)
{
writer.Dispose();
}
}
ms.Seek(0, SeekOrigin.Begin);
ODataEntry entry = null;
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataEntryReader();
while (reader.Read())
{
if (reader.State == ODataReaderState.EntryEnd)
{
entry = (ODataEntry)reader.Item;
}
}
}
Assert.IsTrue(TestHelper.EntryEqual(entry0, entry));
}
public FileWriter(Header header, IAvroWriter <T> writer, long maxBlockCount = 1000)
{
if (header.Schema.ToAvroCanonical() != writer.WriterSchema.ToAvroCanonical())
{
throw new ArgumentException("Incompatible DatumWriter");
}
_header = header;
_writer = writer;
_maxBlockCount = maxBlockCount;
_serializeStream = new MemoryStream(1024 * 1024);
_encoder = new BinaryEncoder(_serializeStream);
_fileStream = _header.FileInfo.Open(FileMode.CreateNew, FileAccess.Write, FileShare.Read);
using (var encoding = new BinaryEncoder(_fileStream))
{
encoding.WriteFixed(_header.Magic);
encoding.WriteMap(_header.Metadata, (s, v) => s.WriteBytes(v));
encoding.WriteFixed(_header.Sync);
}
_fileStream.Flush();
}
public IAvroFileWriter <T> OpenWrite <T>(IAvroWriter <T> datumWriter, long maxBlockCount = 1000)
{
return(new FileWriter <T>(this, datumWriter, maxBlockCount));
}
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
_sw = writer;
IAvroWriter <T> sw1 = writer as IAvroWriter <T>;
ChoGuard.ArgumentNotNull(sw1, "AvroWriter");
using (SequentialWriter <T> sw = new SequentialWriter <T>(sw1, 24))
{
if (records == null)
{
yield break;
}
if (!BeginWrite.Value)
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
object recOutput = null;
var recEnum = records.GetEnumerator();
try
{
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recOutput = record;
if (record != null)
{
if (predicate == null || predicate(record))
{
if (!RaiseBeforeRecordWrite(record, _index, ref recOutput))
{
yield break;
}
if (recOutput == null)
{
continue;
}
else if (!(recOutput is T))
{
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations);
}
if (recOutput is T)
{
sw.Write((T)recOutput);
if (!RaiseAfterRecordWrite(record, _index, recOutput))
{
yield break;
}
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recOutput, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return((T)recOutput);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
}
