protected override IEnumerable <IRow> MutateRow(IRow row)
{
if (row.HasError())
{
var exception = new EtlException(this, "invalid value(s) found");
var index = 0;
foreach (var kvp in row.Values)
{
if (kvp.Value is EtlRowError)
{
var error = kvp.Value as EtlRowError;
exception.Data.Add("Column" + index.ToString("D", CultureInfo.InvariantCulture), kvp.Key);
exception.Data.Add("Value" + index.ToString("D", CultureInfo.InvariantCulture), error.OriginalValue != null
? error.OriginalValue + " (" + error.OriginalValue.GetType().GetFriendlyTypeName() + ")"
: "NULL");
index++;
}
}
exception.Data.Add("Row", row.ToDebugString());
throw exception;
}
yield return(row);
}
protected override IEnumerable <IRow> Produce()
{
var iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.fileRead, FileName, null, null, null, null,
"reading from {FileName}",
PathHelpers.GetFriendlyPathName(FileName));
if (!File.Exists(FileName))
{
var exception = new ProcessExecutionException(this, "input file doesn't exist");
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "input file doesn't exist: {0}",
FileName));
exception.Data.Add("FileName", FileName);
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, 0, exception);
throw exception;
}
var columnConfig = ColumnConfiguration?.ToDictionary(x => x.SourceColumn.ToUpperInvariant(), StringComparer.OrdinalIgnoreCase);
var resultCount = 0;
Stream stream;
StreamReader reader;
try
{
stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.Read);
reader = new StreamReader(stream);
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, null, ex);
var exception = new EtlException(this, "error while opening file", ex);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while opening file: {0}, message: {1}", FileName, ex.Message));
exception.Data.Add("FileName", FileName);
throw exception;
}
var firstRow = true;
var initialValues = new List <KeyValuePair <string, object> >();
var partList = new List <string>(100);
var builder = new StringBuilder(2000);
// capture for performance
var columnNames = ColumnNames;
var delimiter = Delimiter;
var treatEmptyStringAsNull = TreatEmptyStringAsNull;
var removeSurroundingDoubleQuotes = RemoveSurroundingDoubleQuotes;
var throwOnMissingDoubleQuoteClose = ThrowOnMissingDoubleQuoteClose;
var ignoreColumns = IgnoreColumns?.ToHashSet();
try
{
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
string line;
try
{
line = reader.ReadLine();
if (line == null)
{
break;
}
if (string.IsNullOrEmpty(line))
{
continue;
}
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, resultCount, ex);
var exception = new EtlException(this, "error while reading data from file", ex);
exception.Data.Add("FileName", FileName);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while reading data from file: {0}, message: {1}", FileName, ex.Message));
throw exception;
}
if (line.EndsWith(delimiter))
{
line = line[0..^ 1];
public static string FormatExceptionWithDetails(this Exception exception, bool includeTrace = true)
{
try
{
var lvl = 0;
var msg = "EXCEPTION: ";
var cex = exception;
while (cex != null)
{
if (lvl > 0)
{
msg += "\nINNER EXCEPTION: ";
}
msg += cex.GetType().GetFriendlyTypeName() + ": " + cex.Message;
if (cex.Data?["ProcessType"] is string processType)
{
msg += "\n\tPROCESS: ";
if (cex.Data?["ProcessTypeAssembly"] is string processTypeAssembly)
{
msg += "(" + processTypeAssembly + ") ";
}
msg += processType;
if (cex.Data?["ProcessName"] is string processName && processName != processType)
{
msg += " (\"" + processName + "\")";
}
if (cex.Data?["ProcessTopic"] is string processTopic)
{
msg += ", topic: " + processTopic;
}
if (cex.Data?["ProcessKind"] is string processKind)
{
msg += ", kind: " + processKind;
}
}
if (cex.Data?.Count > 0)
{
var first = true;
foreach (var key in cex.Data.Keys)
{
var k = key.ToString();
if (k == "ProcessName" || k == "ProcessKind" || k == "ProcessTopic" || k == "ProcessType" || k == "ProcessTypeAssembly")
{
continue;
}
if (k == "CallChain")
{
continue;
}
if (k == "OpsMessage")
{
continue;
}
if (k == "Trace")
{
continue;
}
if (k == "Row")
{
continue;
}
if (k.Contains("Row", StringComparison.InvariantCultureIgnoreCase) && cex.Data[key] is string rowStr && rowStr.StartsWith("uid", StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
if (first)
{
msg += "\n\tDATA:";
first = false;
}
else
{
msg += ", ";
}
var value = cex.Data[key];
msg += "\n\t\t[" + k + "] = " + (value != null ? value.ToString().Trim() : "NULL");
}
}
if (cex.Data?["Row"] is string storedRow)
{
msg += "\n\tROW: " + storedRow.Replace("\n", "\n\t\t", StringComparison.InvariantCultureIgnoreCase);
}
if (cex.Data?.Count > 0)
{
foreach (var key in cex.Data.Keys)
{
var k = key.ToString();
if (k == "Row")
{
continue;
}
if (k.Contains("Row", StringComparison.InvariantCultureIgnoreCase) && cex.Data[key] is string rowStr && rowStr.StartsWith("uid", StringComparison.InvariantCultureIgnoreCase))
{
msg += "\n\t" + k.ToUpperInvariant() + ": " + rowStr;
}
}
}
if (cex.Data?["CallChain"] is string callChain)
{
msg += "\n\tCALL CHAIN:\n\t\t" + callChain.Replace("\n", "\n\t\t", StringComparison.Ordinal);
}
if (includeTrace)
{
if (cex.Data?["Trace"] is not string trace)
{
trace = EtlException.GetTraceFromStackFrames(new StackTrace(cex, true).GetFrames());
}
if (trace != null)
{
msg += "\n\tTRACE:\n\t\t" + trace.Replace("\n", "\n\t\t", StringComparison.Ordinal);
}
}
cex = cex.InnerException;
lvl++;
}
return(msg);
}
catch (Exception)
{
return(exception.ToString());
}
}