public override IEnumerable<Row> Execute(IEnumerable<Row> rows) { Logger.Info("Reading Fixed {0}", _name); if (_top > 0) { using (var file = new FluentFile(_classBuilder.CreateRecordClass()).From(_fullName).OnError(_errorMode)) { foreach (var obj in file) { var row = Row.FromObject(obj); row["TflFileName"] = _fullName; if (_counter < _top) { Interlocked.Increment(ref _counter); yield return row; } else { yield break; } } HandleErrors(file); } } else { using (var file = new FluentFile(_classBuilder.CreateRecordClass()).From(_fullName).OnError(_errorMode)) { foreach (var obj in file) { var row = Row.FromObject(obj); row["TflFileName"] = _fullName; yield return row; } HandleErrors(file); } } }
/// <summary> /// Notifies the observer of a new value in the sequence. It's best to override Dispatch or TreatRow than this method because this method contains pipelining logic. /// </summary> public override void Trigger() { CountTreated++; try { IEnumerator fList = null; if (_strm != null) { using (StreamReader reader = new StreamReader(_strm)) { fList = FluentFile.For(_type).From(reader).GetEnumerator(); } } else if (_strmReader != null) { fList = FluentFile.For(_type).From(_strmReader).GetEnumerator(); } else if (_filename != null) { fList = FluentFile.For(_type).From(_filename).GetEnumerator(); } IterateElements(fList); } catch (Exception ex) { log4net.LogManager.GetLogger(this.GetType()).Error("Operation error", ex); Observers.PropagateOnError(ex); } Completed = true; Observers.PropagateOnCompleted(); }
public override IEnumerable <Row> Execute(IEnumerable <Row> rows) { PrepareType(_entity); FluentFile engine; var inserts = 0; try { engine = new FluentFile(Type) { Encoding = _connection.Encoding.Equals("utf-8w/obom") ? new UTF8Encoding(false) : Encoding.GetEncoding(_connection.Encoding) }; } catch (Exception ex) { throw new TransformalizeException(Logger, EntityName, ex.Message); } if (!_connection.Header.Equals(string.Empty)) { PrepareHeader(_entity); engine.HeaderText = HeaderText; } if (!_connection.Footer.Equals(string.Empty)) { PrepareFooter(_entity); engine.FooterText = FooterText; } using (var file = engine.To(FileInfo.FullName)) { foreach (var row in rows) { foreach (var field in _stringFields) { var value = row[field].ToString(); if (_isCsv) { row[field] = value.Replace("\r\n", "\n"); } else if (_connection.Delimiter != SPACE) { row[field] = value.Replace(_connection.Delimiter, SPACE); } } foreach (var field in _mapFields) { row[field.Identifier] = row[field.Alias]; } var record = row.ToObject(Type); file.Write(record); inserts++; } } _entity.Inserts = inserts; yield break; }
public override IEnumerable <Row> Execute(IEnumerable <Row> rows) { using (FileEngine file = FluentFile.For <UserAddressRecord>().From(filePath)) { foreach (object obj in file) { yield return(Row.FromObject(obj)); } } }
public override IEnumerable <Row> Execute(IEnumerable <Row> rows) { using (var file = FluentFile.For <DataRecord>().From(FilePath)) { foreach (var obj in file) { yield return(Row.FromObject(obj)); } } }
protected override async Task ExecuteYield(IAsyncEnumerable <Row> rows, AsyncEnumerator <Row> .Yield yield, CancellationToken cancellationToken = default) { using (FileEngine file = FluentFile.For <UserRecord>().From("users.txt")) { foreach (object obj in file) { await @yield.ReturnAsync(Row.FromObject(obj)); } } }
public override IEnumerable <Row> Execute(IEnumerable <Row> rows) { using (FileEngine file = FluentFile.For <FlatFile>().From(FilePath)) { int i = 0; foreach (FlatFile fileRow in file) { yield return(Row.FromObject(fileRow)); i++; } } }
public override IEnumerable <Row> Execute(IEnumerable <Row> rows) { FluentFile engine = FluentFile.For <UserFullRecord>(); engine.HeaderText = "Id\tName\tAddress"; using (FileEngine file = engine.To(filePath)) { foreach (Row row in rows) { file.Write(row.ToObject <UserFullRecord>()); //pass through rows if needed for another later operation yield return(row); } } }
/// <summary> /// Executes this operation /// </summary> /// <param name="rows">The rows.</param> /// <returns></returns> public override IEnumerable <Row> Execute(IEnumerable <Row> rows) { FluentFile engine = FluentFile.For <UserRecord>(); engine.HeaderText = "Id\tName\tEmail"; using (FileEngine file = engine.To("users.txt")) { foreach (Row row in rows) { UserRecord record = new UserRecord(); record.Id = (int)row["id"]; record.Name = (string)row["name"]; record.Email = (string)row["email"]; file.Write(record); } } yield break; }
protected override async Task ExecuteYield(IAsyncEnumerable <Row> rows, AsyncEnumerator <Row> .Yield yield, CancellationToken cancellationToken = default) { FluentFile engine = FluentFile.For <UserRecord>(); engine.HeaderText = "Id\tName\tEmail"; using (FileEngine file = engine.To("users.txt")) { await rows.ForEachAsync(row => { UserRecord record = new UserRecord(); record.Id = (int)row["id"]; record.Name = (string)row["name"]; record.Email = (string)row["email"]; file.Write(record); }, cancellationToken); } @yield.Break(); }
/// <summary> /// Initialize the file engine /// </summary> /// <returns>initialized engine</returns> public FileEngine InitializeEngine() { if (Engine != null) { return(Engine); } if (this.Stream != null && _innerstrmwriter == null) { _innerstrmwriter = new StreamWriter(this.Stream); } var ff = FluentFile.For <T>(); if (PrepareFluentFile != null) { PrepareFluentFile(ff); } if (_innerstrmwriter != null) { Engine = ff.To(_innerstrmwriter); } else if (Writer != null) { Engine = ff.To(Writer); } else if (FileName != null) { Engine = ff.To(FileName); } if (Engine == null) { throw new InvalidOperationException("File write is not initialized appropriately"); } return(Engine); }
public override IEnumerable <Row> Execute(IEnumerable <Row> rows) { Logger.Info("Reading Fixed {0}", _name); if (_top > 0) { using (var file = new FluentFile(_classBuilder.CreateRecordClass()).From(_fullName).OnError(_errorMode)) { foreach (var obj in file) { var row = Row.FromObject(obj); row["TflFileName"] = _fullName; if (_counter < _top) { Interlocked.Increment(ref _counter); yield return(row); } else { yield break; } } HandleErrors(file); } } else { using (var file = new FluentFile(_classBuilder.CreateRecordClass()).From(_fullName).OnError(_errorMode)) { foreach (var obj in file) { var row = Row.FromObject(obj); row["TflFileName"] = _fullName; yield return(row); } HandleErrors(file); } } }
public override IEnumerable<Row> Execute(IEnumerable<Row> rows) { PrepareType(_entity); FluentFile engine; var inserts = 0; try { engine = new FluentFile(Type) { Encoding = _connection.Encoding.Equals("utf-8w/obom") ? new UTF8Encoding(false) : Encoding.GetEncoding(_connection.Encoding) }; } catch (Exception ex) { throw new TransformalizeException(Logger, EntityName, ex.Message); } if (!_connection.Header.Equals(string.Empty)) { PrepareHeader(_entity); engine.HeaderText = HeaderText; } if (!_connection.Footer.Equals(string.Empty)) { PrepareFooter(_entity); engine.FooterText = FooterText; } using (var file = engine.To(FileInfo.FullName)) { foreach (var row in rows) { foreach (var field in _stringFields) { var value = row[field].ToString(); if (_isCsv) { row[field] = value.Replace("\r\n", "\n"); } else if (_connection.Delimiter != SPACE) { row[field] = value.Replace(_connection.Delimiter, SPACE); } } foreach (var field in _mapFields) { row[field.Identifier] = row[field.Alias]; } var record = row.ToObject(Type); file.Write(record); inserts++; } } _entity.Inserts = inserts; yield break; }
public override IEnumerable <Row> Execute(IEnumerable <Row> rows) { var cb = new DelimitedClassBuilder("Tfl" + _entity.OutputName()) { IgnoreEmptyLines = true, Delimiter = _delimiter.ToString(CultureInfo.InvariantCulture), IgnoreFirstLines = _ignoreFirstLines }; foreach (var field in _fields) { if (field.IsQuoted()) { cb.AddField(new DelimitedFieldBuilder(field.Identifier, typeof(string)) { FieldQuoted = true, QuoteChar = field.QuotedWith, QuoteMode = QuoteMode.OptionalForRead, FieldOptional = field.Optional }); } else { cb.AddField(new DelimitedFieldBuilder(field.Identifier, typeof(string)) { FieldOptional = field.Optional }); } } Logger.Info("Reading Delimited {0}", _name); var conversionMap = Common.GetObjectConversionMap(); if (_top > 0) { using (var file = new FluentFile(cb.CreateRecordClass()).From(_fullName).OnError(_errorMode)) { foreach (var row in from object obj in file select Row.FromObject(obj)) { if (_counter == 0) { try { ProcessRow(row, _fields, _fullName, conversionMap); } catch (Exception ex) { if (!_warned) { Logger.EntityWarn(_entity.Alias, "First row from '{0}' failed to process. You may have headers. If so, set start=\"2\" on connection. Note: start is 1-based. {0}", _name, ex.Message); _warned = true; } } } else { ProcessRow(row, _fields, _fullName, conversionMap); } if (_counter < _top) { yield return(row); } else { yield break; } } Interlocked.Increment(ref _counter); HandleErrors(file); } } else { using (var file = new FluentFile(cb.CreateRecordClass()).From(_fullName).OnError(_errorMode)) { foreach (var row in from object obj in file select Row.FromObject(obj)) { if (_counter == 0) { try { ProcessRow(row, _fields, _fullName, conversionMap); } catch (Exception ex) { Logger.EntityWarn(_entity.Alias, "First row failed to process. You may have headers. If so, set start-index=\"2\" on connection. {0}", ex.Message); } } else { ProcessRow(row, _fields, _fullName, conversionMap); } yield return(row); Interlocked.Increment(ref _counter); } HandleErrors(file); } } }
public override IEnumerable<Row> Execute(IEnumerable<Row> rows) { var cb = new DelimitedClassBuilder("Tfl" + _entity.OutputName()) { IgnoreEmptyLines = true, Delimiter = _delimiter.ToString(CultureInfo.InvariantCulture), IgnoreFirstLines = _ignoreFirstLines }; foreach (var field in _fields) { if (field.IsQuoted()) { cb.AddField(new DelimitedFieldBuilder(field.Identifier, typeof(string)) { FieldQuoted = true, QuoteChar = field.QuotedWith, QuoteMode = QuoteMode.OptionalForRead, FieldOptional = field.Optional }); } else { cb.AddField(new DelimitedFieldBuilder(field.Identifier, typeof(string)) { FieldOptional = field.Optional }); } } Logger.Info("Reading Delimited {0}", _name); var conversionMap = Common.GetObjectConversionMap(); if (_top > 0) { using (var file = new FluentFile(cb.CreateRecordClass()).From(_fullName).OnError(_errorMode)) { foreach (var row in from object obj in file select Row.FromObject(obj)) { if (_counter == 0) { try { ProcessRow(row, _fields, _fullName, conversionMap); } catch (Exception ex) { if (!_warned) { Logger.EntityWarn(_entity.Alias, "First row from '{0}' failed to process. You may have headers. If so, set start=\"2\" on connection. Note: start is 1-based. {0}", _name, ex.Message); _warned = true; } } } else { ProcessRow(row, _fields, _fullName, conversionMap); } if (_counter < _top) { yield return row; } else { yield break; } } Interlocked.Increment(ref _counter); HandleErrors(file); } } else { using (var file = new FluentFile(cb.CreateRecordClass()).From(_fullName).OnError(_errorMode)) { foreach (var row in from object obj in file select Row.FromObject(obj)) { if (_counter == 0) { try { ProcessRow(row, _fields, _fullName, conversionMap); } catch (Exception ex) { Logger.EntityWarn(_entity.Alias, "First row failed to process. You may have headers. If so, set start-index=\"2\" on connection. {0}", ex.Message); } } else { ProcessRow(row, _fields, _fullName, conversionMap); } yield return row; Interlocked.Increment(ref _counter); } HandleErrors(file); } } }