protected override IEnumerable <IRow> Produce()
{
var iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.fileRead, FileName, null, null, null, null,
"reading from {FileName}",
PathHelpers.GetFriendlyPathName(FileName));
if (!File.Exists(FileName))
{
var exception = new ProcessExecutionException(this, "input file doesn't exist");
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "input file doesn't exist: {0}",
FileName));
exception.Data.Add("FileName", FileName);
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, 0, exception);
throw exception;
}
var columnConfig = ColumnConfiguration?.ToDictionary(x => x.SourceColumn.ToUpperInvariant(), StringComparer.OrdinalIgnoreCase);
var resultCount = 0;
Stream stream;
StreamReader reader;
try
{
stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.Read);
reader = new StreamReader(stream);
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, null, ex);
var exception = new EtlException(this, "error while opening file", ex);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while opening file: {0}, message: {1}", FileName, ex.Message));
exception.Data.Add("FileName", FileName);
throw exception;
}
var firstRow = true;
var initialValues = new List <KeyValuePair <string, object> >();
var partList = new List <string>(100);
var builder = new StringBuilder(2000);
// capture for performance
var columnNames = ColumnNames;
var delimiter = Delimiter;
var treatEmptyStringAsNull = TreatEmptyStringAsNull;
var removeSurroundingDoubleQuotes = RemoveSurroundingDoubleQuotes;
var throwOnMissingDoubleQuoteClose = ThrowOnMissingDoubleQuoteClose;
var ignoreColumns = IgnoreColumns?.ToHashSet();
try
{
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
string line;
try
{
line = reader.ReadLine();
if (line == null)
{
break;
}
if (string.IsNullOrEmpty(line))
{
continue;
}
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, resultCount, ex);
var exception = new EtlException(this, "error while reading data from file", ex);
exception.Data.Add("FileName", FileName);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while reading data from file: {0}, message: {1}", FileName, ex.Message));
throw exception;
}
if (line.EndsWith(delimiter))
{
line = line[0..^ 1];
protected override IEnumerable <IRow> Produce()
{
var inputRows = InputGenerator.Invoke(this);
if (ColumnConfiguration != null)
{
var initialValues = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
if (CopyOnlySpecifiedColumns)
{
foreach (var row in inputRows)
{
if (Context.CancellationTokenSource.IsCancellationRequested)
{
yield break;
}
foreach (var config in ColumnConfiguration)
{
var value = HandleConverter(row[config.SourceColumn], config);
initialValues[config.RowColumn ?? config.SourceColumn] = value;
}
var newRow = Context.CreateRow(this, initialValues);
newRow.Tag = row.Tag;
yield return(newRow);
initialValues.Clear();
}
}
else
{
var columnConfig = ColumnConfiguration.ToDictionary(x => x.SourceColumn.ToUpperInvariant());
foreach (var row in inputRows)
{
if (Context.CancellationTokenSource.IsCancellationRequested)
{
yield break;
}
foreach (var config in ColumnConfiguration)
{
var value = HandleConverter(row[config.SourceColumn], config);
initialValues[config.RowColumn ?? config.SourceColumn] = value;
}
foreach (var kvp in row.Values)
{
if (!columnConfig.ContainsKey(kvp.Key.ToUpperInvariant()))
{
if (DefaultColumnConfiguration != null)
{
var value = HandleConverter(kvp.Value, DefaultColumnConfiguration);
initialValues[kvp.Key] = value;
}
else
{
initialValues[kvp.Key] = kvp.Value;
}
}
}
var newRow = Context.CreateRow(this, initialValues);
newRow.Tag = row.Tag;
yield return(newRow);
initialValues.Clear();
}
}
}
else
{
foreach (var row in inputRows)
{
if (Context.CancellationTokenSource.IsCancellationRequested)
{
yield break;
}
yield return(Context.CreateRow(this, row));
}
}
}