public void Write(T record) { if (record is DataTable) { Write(record as DataTable); return; } else if (record is IDataReader) { Write(record as IDataReader); return; } if (_streamWriter != null) { _avroWriter = Create(_streamWriter.Value); } _writer.Writer = this; _writer.TraceSwitch = TraceSwitch; if (record is ArrayList) { _writer.WriteTo <T>(_avroWriter, ((IEnumerable)record).AsTypedEnumerable <object>()).Loop(); } else if (record != null && !(/*!record.GetType().IsDynamicType() && record is IDictionary*/ record.GetType() == typeof(ExpandoObject) || typeof(IDynamicMetaObjectProvider).IsAssignableFrom(record.GetType()) || record.GetType() == typeof(object) || record.GetType().IsAnonymousType()) && (typeof(IDictionary).IsAssignableFrom(record.GetType()) || (record.GetType().IsGenericType&& record.GetType().GetGenericTypeDefinition() == typeof(IDictionary <,>)))) { _writer.WriteTo <T>(_avroWriter, ((IEnumerable)record).AsTypedEnumerable <object>()).Loop(); } else { _writer.WriteTo <T>(_avroWriter, new object[] { record }).Loop(); } }
protected ChoAvroWriter(IAvroWriter <Dictionary <string, object> > avroWriter, ChoAvroRecordConfiguration configuration = null) { ChoGuard.ArgumentNotNull(avroWriter, "AvroWriter"); Configuration = configuration; Init(); _avroWriter = avroWriter; }
public ChoAvroWriter(IAvroWriter <T> avroWriter, ChoAvroRecordConfiguration configuration = null) { ChoGuard.ArgumentNotNull(avroWriter, "AvroWriter"); Configuration = configuration; Init(); _avroWriter = avroWriter; }
public void Write(IEnumerable <T> records) { if (_streamWriter != null) { _avroWriter = Create(_streamWriter.Value); } _writer.Writer = this; _writer.TraceSwitch = TraceSwitch; _writer.WriteTo <T>(_avroWriter, records.OfType <object>()).Loop(); }
public override void WriteAvroFile(Stream stream) { using (IAvroWriter <object> streamWriter = AvroContainer.CreateGenericWriter(this.GetSchema().ToString(), stream, this.Codec)) { using (var sequentialWriter = new SequentialWriter <object>(streamWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize)) { foreach (LargeClass record in this.Expected) { sequentialWriter.Write(record.ToAvroRecord(this.GetSchema())); } } } }
/// <summary> /// Initializes a new instance of the <see cref="SequentialWriter{T}"/> class. /// </summary> /// <param name="writer">The writer.</param> /// <param name="syncNumberOfObjects">The sync number of objects.</param> /// <exception cref="System.ArgumentNullException">Thrown when <paramref name="writer"/> is null.</exception> public SequentialWriter(IAvroWriter <T> writer, int syncNumberOfObjects) { if (writer == null) { throw new ArgumentNullException("writer"); } if (syncNumberOfObjects <= 0) { throw new ArgumentOutOfRangeException("syncNumberOfObjects"); } this.writer = writer; this.metadata = new Dictionary <string, byte[]>(); this.metadataWritten = false; this.current = writer.CreateBlock(); this.syncNumberOfObjects = syncNumberOfObjects; }
public override void WriteAvroFile(Stream stream) { IAvroWriter <LargeClass> avroWriter = AvroContainer.CreateWriter <LargeClass>( stream, new AvroSerializerSettings { UsePosixTime = true }, this.Codec); using (var writer = new SequentialWriter <LargeClass>(avroWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize)) { foreach (LargeClass record in this.Expected) { writer.Write(record); } } }
public void TestReadFeed() { MemoryStream ms = new MemoryStream(); IAvroWriter <Product[]> writer = null; try { writer = AvroContainer.CreateWriter <Product[]>(ms, true, new AvroSerializerSettings(), Codec.Null); using (var seqWriter = new SequentialWriter <Product[]>(writer, 24)) { seqWriter.Write(new[] { product1, product2, product3 }); } } finally { if (writer != null) { writer.Dispose(); } } ms.Seek(0, SeekOrigin.Begin); var entries = new List <ODataEntry>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataFeedReader(); while (reader.Read()) { if (reader.State == ODataReaderState.EntryEnd) { entries.Add((ODataEntry)reader.Item); } } } Assert.AreEqual(3, entries.Count); Assert.IsTrue(TestHelper.EntryEqual(entry1, entries[0])); Assert.IsTrue(TestHelper.EntryEqual(entry2, entries[1])); Assert.IsTrue(TestHelper.EntryEqual(entry3, entries[2])); }
public void TestReadEntry() { MemoryStream ms = new MemoryStream(); IAvroWriter <Product> writer = null; try { writer = AvroContainer.CreateWriter <Product>(ms, true, new AvroSerializerSettings(), Codec.Null); using (var seqWriter = new SequentialWriter <Product>(writer, 24)) { seqWriter.Write(product0); } } finally { if (writer != null) { writer.Dispose(); } } ms.Seek(0, SeekOrigin.Begin); ODataEntry entry = null; using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataEntryReader(); while (reader.Read()) { if (reader.State == ODataReaderState.EntryEnd) { entry = (ODataEntry)reader.Item; } } } Assert.IsTrue(TestHelper.EntryEqual(entry0, entry)); }
public FileWriter(Header header, IAvroWriter <T> writer, long maxBlockCount = 1000) { if (header.Schema.ToAvroCanonical() != writer.WriterSchema.ToAvroCanonical()) { throw new ArgumentException("Incompatible DatumWriter"); } _header = header; _writer = writer; _maxBlockCount = maxBlockCount; _serializeStream = new MemoryStream(1024 * 1024); _encoder = new BinaryEncoder(_serializeStream); _fileStream = _header.FileInfo.Open(FileMode.CreateNew, FileAccess.Write, FileShare.Read); using (var encoding = new BinaryEncoder(_fileStream)) { encoding.WriteFixed(_header.Magic); encoding.WriteMap(_header.Metadata, (s, v) => s.WriteBytes(v)); encoding.WriteFixed(_header.Sync); } _fileStream.Flush(); }
public IAvroFileWriter <T> OpenWrite <T>(IAvroWriter <T> datumWriter, long maxBlockCount = 1000) { return(new FileWriter <T>(this, datumWriter, maxBlockCount)); }
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { _sw = writer; IAvroWriter <T> sw1 = writer as IAvroWriter <T>; ChoGuard.ArgumentNotNull(sw1, "AvroWriter"); using (SequentialWriter <T> sw = new SequentialWriter <T>(sw1, 24)) { if (records == null) { yield break; } if (!BeginWrite.Value) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; object recOutput = null; var recEnum = records.GetEnumerator(); try { object notNullRecord = GetFirstNotNullRecord(recEnum); if (notNullRecord == null) { yield break; } foreach (object record in GetRecords(recEnum)) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recOutput = record; if (record != null) { if (predicate == null || predicate(record)) { if (!RaiseBeforeRecordWrite(record, _index, ref recOutput)) { yield break; } if (recOutput == null) { continue; } else if (!(recOutput is T)) { continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations); } if (recOutput is T) { sw.Write((T)recOutput); if (!RaiseAfterRecordWrite(record, _index, recOutput)) { yield break; } } } catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recOutput, ex)) { throw; } else { //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } } yield return((T)recOutput); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } } }