protected override IEnumerable <IRow> MutateRow(IRow row) { if (row.HasError()) { var exception = new EtlException(this, "invalid value(s) found"); var index = 0; foreach (var kvp in row.Values) { if (kvp.Value is EtlRowError) { var error = kvp.Value as EtlRowError; exception.Data.Add("Column" + index.ToString("D", CultureInfo.InvariantCulture), kvp.Key); exception.Data.Add("Value" + index.ToString("D", CultureInfo.InvariantCulture), error.OriginalValue != null ? error.OriginalValue + " (" + error.OriginalValue.GetType().GetFriendlyTypeName() + ")" : "NULL"); index++; } } exception.Data.Add("Row", row.ToDebugString()); throw exception; } yield return(row); }
protected override IEnumerable <IRow> Produce() { var iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.fileRead, FileName, null, null, null, null, "reading from {FileName}", PathHelpers.GetFriendlyPathName(FileName)); if (!File.Exists(FileName)) { var exception = new ProcessExecutionException(this, "input file doesn't exist"); exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "input file doesn't exist: {0}", FileName)); exception.Data.Add("FileName", FileName); Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, 0, exception); throw exception; } var columnConfig = ColumnConfiguration?.ToDictionary(x => x.SourceColumn.ToUpperInvariant(), StringComparer.OrdinalIgnoreCase); var resultCount = 0; Stream stream; StreamReader reader; try { stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.Read); reader = new StreamReader(stream); } catch (Exception ex) { Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, null, ex); var exception = new EtlException(this, "error while opening file", ex); exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while opening file: {0}, message: {1}", FileName, ex.Message)); exception.Data.Add("FileName", FileName); throw exception; } var firstRow = true; var initialValues = new List <KeyValuePair <string, object> >(); var partList = new List <string>(100); var builder = new StringBuilder(2000); // capture for performance var columnNames = ColumnNames; var delimiter = Delimiter; var treatEmptyStringAsNull = TreatEmptyStringAsNull; var removeSurroundingDoubleQuotes = RemoveSurroundingDoubleQuotes; var throwOnMissingDoubleQuoteClose = ThrowOnMissingDoubleQuoteClose; var ignoreColumns = IgnoreColumns?.ToHashSet(); try { while (!Context.CancellationTokenSource.IsCancellationRequested) { string line; try { line = reader.ReadLine(); if (line == null) { break; } if (string.IsNullOrEmpty(line)) { continue; } } catch (Exception ex) { Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, resultCount, ex); var exception = new EtlException(this, "error while reading data from file", ex); exception.Data.Add("FileName", FileName); exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while reading data from file: {0}, message: {1}", FileName, ex.Message)); throw exception; } if (line.EndsWith(delimiter)) { line = line[0..^ 1];
public static string FormatExceptionWithDetails(this Exception exception, bool includeTrace = true) { try { var lvl = 0; var msg = "EXCEPTION: "; var cex = exception; while (cex != null) { if (lvl > 0) { msg += "\nINNER EXCEPTION: "; } msg += cex.GetType().GetFriendlyTypeName() + ": " + cex.Message; if (cex.Data?["ProcessType"] is string processType) { msg += "\n\tPROCESS: "; if (cex.Data?["ProcessTypeAssembly"] is string processTypeAssembly) { msg += "(" + processTypeAssembly + ") "; } msg += processType; if (cex.Data?["ProcessName"] is string processName && processName != processType) { msg += " (\"" + processName + "\")"; } if (cex.Data?["ProcessTopic"] is string processTopic) { msg += ", topic: " + processTopic; } if (cex.Data?["ProcessKind"] is string processKind) { msg += ", kind: " + processKind; } } if (cex.Data?.Count > 0) { var first = true; foreach (var key in cex.Data.Keys) { var k = key.ToString(); if (k == "ProcessName" || k == "ProcessKind" || k == "ProcessTopic" || k == "ProcessType" || k == "ProcessTypeAssembly") { continue; } if (k == "CallChain") { continue; } if (k == "OpsMessage") { continue; } if (k == "Trace") { continue; } if (k == "Row") { continue; } if (k.Contains("Row", StringComparison.InvariantCultureIgnoreCase) && cex.Data[key] is string rowStr && rowStr.StartsWith("uid", StringComparison.InvariantCultureIgnoreCase)) { continue; } if (first) { msg += "\n\tDATA:"; first = false; } else { msg += ", "; } var value = cex.Data[key]; msg += "\n\t\t[" + k + "] = " + (value != null ? value.ToString().Trim() : "NULL"); } } if (cex.Data?["Row"] is string storedRow) { msg += "\n\tROW: " + storedRow.Replace("\n", "\n\t\t", StringComparison.InvariantCultureIgnoreCase); } if (cex.Data?.Count > 0) { foreach (var key in cex.Data.Keys) { var k = key.ToString(); if (k == "Row") { continue; } if (k.Contains("Row", StringComparison.InvariantCultureIgnoreCase) && cex.Data[key] is string rowStr && rowStr.StartsWith("uid", StringComparison.InvariantCultureIgnoreCase)) { msg += "\n\t" + k.ToUpperInvariant() + ": " + rowStr; } } } if (cex.Data?["CallChain"] is string callChain) { msg += "\n\tCALL CHAIN:\n\t\t" + callChain.Replace("\n", "\n\t\t", StringComparison.Ordinal); } if (includeTrace) { if (cex.Data?["Trace"] is not string trace) { trace = EtlException.GetTraceFromStackFrames(new StackTrace(cex, true).GetFrames()); } if (trace != null) { msg += "\n\tTRACE:\n\t\t" + trace.Replace("\n", "\n\t\t", StringComparison.Ordinal); } } cex = cex.InnerException; lvl++; } return(msg); } catch (Exception) { return(exception.ToString()); } }