protected override void ExecuteImpl()
{
if (!File.Exists(FileName))
{
Context.Log(LogSeverity.Debug, this, "can't delete file because it doesn't exist '{FileName}'", PathHelpers.GetFriendlyPathName(FileName));
return;
}
Context.Log(LogSeverity.Information, this, "deleting file '{FileName}'", PathHelpers.GetFriendlyPathName(FileName));
try
{
File.Delete(FileName);
Context.Log(LogSeverity.Debug, this, "successfully deleted file '{FileName}' in {Elapsed}", PathHelpers.GetFriendlyPathName(FileName),
InvocationInfo.LastInvocationStarted.Elapsed);
}
catch (Exception ex)
{
var exception = new ProcessExecutionException(this, "file deletion failed", ex);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "file deletion failed, file name: {0}, message: {1}",
FileName, ex.Message));
exception.Data.Add("FileName", FileName);
throw exception;
}
}
protected override void ExecuteImpl()
{
using (var clt = new WebClient())
{
var iocUid = 0;
try
{
using (Context.CancellationTokenSource.Token.Register(clt.CancelAsync))
{
iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.httpGet, Url, null, null, null, null,
"downloading file from {Url} to {FileName}",
Url, PathHelpers.GetFriendlyPathName(FileName));
clt.DownloadFile(Url, FileName);
Context.RegisterIoCommandSuccess(this, IoCommandKind.httpGet, iocUid, Convert.ToInt32(new FileInfo(FileName).Length));
}
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.httpGet, iocUid, null, ex);
var exception = new ProcessExecutionException(this, "file download failed", ex);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "file download failed, url: {0}, file name: {1}, message: {2}",
Url, FileName, ex.Message));
exception.Data.Add("Url", Url);
exception.Data.Add("FileName", FileName);
throw exception;
}
}
}
protected override IEnumerable <IRow> Produce()
{
var iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.fileRead, FileName, null, null, null, null,
"reading from {FileName}",
PathHelpers.GetFriendlyPathName(FileName));
if (!File.Exists(FileName))
{
var exception = new ProcessExecutionException(this, "input file doesn't exist");
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "input file doesn't exist: {0}",
FileName));
exception.Data.Add("FileName", FileName);
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, 0, exception);
throw exception;
}
var columnConfig = ColumnConfiguration?.ToDictionary(x => x.SourceColumn.ToUpperInvariant(), StringComparer.OrdinalIgnoreCase);
var resultCount = 0;
Stream stream;
StreamReader reader;
try
{
stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.Read);
reader = new StreamReader(stream);
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, null, ex);
var exception = new EtlException(this, "error while opening file", ex);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while opening file: {0}, message: {1}", FileName, ex.Message));
exception.Data.Add("FileName", FileName);
throw exception;
}
var firstRow = true;
var initialValues = new List <KeyValuePair <string, object> >();
var partList = new List <string>(100);
var builder = new StringBuilder(2000);
// capture for performance
var columnNames = ColumnNames;
var delimiter = Delimiter;
var treatEmptyStringAsNull = TreatEmptyStringAsNull;
var removeSurroundingDoubleQuotes = RemoveSurroundingDoubleQuotes;
var throwOnMissingDoubleQuoteClose = ThrowOnMissingDoubleQuoteClose;
var ignoreColumns = IgnoreColumns?.ToHashSet();
try
{
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
string line;
try
{
line = reader.ReadLine();
if (line == null)
{
break;
}
if (string.IsNullOrEmpty(line))
{
continue;
}
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, resultCount, ex);
var exception = new EtlException(this, "error while reading data from file", ex);
exception.Data.Add("FileName", FileName);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while reading data from file: {0}, message: {1}", FileName, ex.Message));
throw exception;
}
if (line.EndsWith(delimiter))
{
line = line[0..^ 1];
