private void LogValidationErrorMessage
(
string errorMessage,
long errorRecordIndex,
int?errorFieldIndex,
string errorFieldName,
long?errorFilePosition
)
{
var errorDateTime = DateTime.Now;
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = _context.EtlPackageId,
EtlSessionId = _context.EtlSessionId,
EtlStepName = _step.Name,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = errorMessage,
Flags = errorRecordIndex,
StackTrace = BuildErrorMessageStackTrace
(
errorRecordIndex,
errorFieldIndex,
errorFieldName,
errorFilePosition
),
}
);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException("Source.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException("Source.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.Text))
{
throw new InvalidOperationException("Source.Text cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.ConnectionString))
{
throw new InvalidOperationException("Destination.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.ProviderName))
{
throw new InvalidOperationException("Destination.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.TableName))
{
throw new InvalidOperationException("Destination.TableName cannot be empty");
}
if (this.Mappings.Count == 0)
{
throw new InvalidOperationException("Mappings not specified");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var insertedRowCount = 0L;
var errorRowCount = 0L;
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
{
var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
{
var errorDateTime = DateTime.Now;
result.Status = EtlStatus.FinishedWithLosses;
result.Message = e.Message;
errorRowCount++;
logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EtlStepName = this.Name,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = e.Message,
Flags = e.RecordIndex,
StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
}
);
if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
{
result.Status = EtlStatus.FinishedWithLosses;
e.TrySkipError = true;
}
else
{
result.Status = EtlStatus.Failed;
}
};
insertedRowCount = dbWriter.Write(mapReader, this.TimeoutMilliseconds, this.BatchSize);
}
}
}
sourceRowCount = insertedRowCount + errorRowCount;
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Found",
// Flags = sourceRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Inserted",
// Flags = insertedRowCount,
//});
return(result);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException("Source.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException("Source.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.Text))
{
throw new InvalidOperationException("Source.Text cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.ConnectionString))
{
throw new InvalidOperationException("Destination.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.ProviderName))
{
throw new InvalidOperationException("Destination.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.TableName))
{
throw new InvalidOperationException("Destination.TableName cannot be empty");
}
if (this.Mappings.Count == 0)
{
throw new InvalidOperationException("Mappings not specified");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var insertedRowCount = 0L;
var errorRowCount = 0L;
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
{
var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
{
var errorDateTime = DateTime.Now;
result.Status = EtlStatus.FinishedWithLosses;
result.Message = e.Message;
errorRowCount++;
logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EtlStepName = this.Name,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = e.Message,
Flags = e.RecordIndex,
StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
}
);
if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
{
result.Status = EtlStatus.FinishedWithLosses;
e.TrySkipError = true;
}
else
{
result.Status = EtlStatus.Failed;
}
};
insertedRowCount = dbWriter.Write(mapReader, this.TimeoutMilliseconds, this.BatchSize);
}
}
}
sourceRowCount = insertedRowCount + errorRowCount;
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Found",
// Flags = sourceRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Inserted",
// Flags = insertedRowCount,
//});
return result;
}
private EtlSession InvokePackageCore(EtlPackage package, EtlVariableAssignment[] assignments, string parentSessionId)
{
var session = new EtlSession
{
EtlSessionId = Guid.NewGuid().ToString(),
EtlPackageId = package.Id,
EtlPackageName = package.Name,
ParentEtlSessionId = parentSessionId,
Status = EtlStatus.Started,
UserName = GetCurrentUserName(),
};
session.StartDateTime = DateTime.Now;
session.StartUtcDateTime = session.StartDateTime.ToUniversalTime();
_logger.LogEtlSessionStart(session);
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = session.EtlPackageId,
EtlSessionId = session.EtlSessionId,
LogDateTime = session.StartDateTime,
LogUtcDateTime = session.StartUtcDateTime,
MessageType = EtlMessageType.SessionStart,
Text = string.Format(Properties.Resources.SessionStarted, session.EtlSessionId, session.EtlPackageId, session.EtlPackageName)
}
);
try
{
InvokePackageCatched(package, session, assignments);
}
catch (Exception exc)
{
if (IsCriticalException(exc))
{
throw;
}
session.Status = EtlStatus.Failed;
var errorDateTime = DateTime.Now;
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = session.EtlPackageId,
EtlSessionId = session.EtlSessionId,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = exc.Message,
StackTrace = exc.StackTrace,
}
);
}
finally
{
session.EndDateTime = DateTime.Now;
session.EndUtcDateTime = session.EndDateTime.Value.ToUniversalTime();
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = session.EtlPackageId,
EtlSessionId = session.EtlSessionId,
LogDateTime = session.EndDateTime.Value,
LogUtcDateTime = session.EndUtcDateTime.Value,
MessageType = EtlMessageType.SessionEnd,
Text = string.Format(Properties.Resources.SessionFinished, session.EtlSessionId, session.EtlPackageId, session.EtlPackageName, session.Status)
}
);
_logger.LogEtlSessionEnd(session);
}
return(session);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
}
if (this.Destination == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
}
if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
}
if (string.IsNullOrEmpty(this.Destination.ConnectionString))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
}
if (string.IsNullOrEmpty(this.Destination.ProviderName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ProviderName"));
}
if (string.IsNullOrEmpty(this.Destination.TableName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.TableName"));
}
if (this.Source.FieldDelimiter.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
}
if (this.Source.Quote != null && this.Source.Quote.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
}
if (this.Source.Escape != null && this.Source.Escape.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var insertedRowCount = 0L;
var errorRowCount = 0L;
var csvSyntax = new CsvSyntaxInfo
{
HasHeaders = this.Source.HasHeaders,
FieldDelimiter = this.Source.FieldDelimiter[0],
Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
};
using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
{
using (var csvReader = new CsvReader(fileReader, csvSyntax))
{
var wrappedReader = new EtlMappedDataReader(csvReader, this.Mappings);
var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
{
var errorDateTime = DateTime.Now;
result.Status = EtlStatus.FinishedWithLosses;
result.Message = e.Message;
//todo: fix error counter in CSV import. Now it counts errors, not error records
errorRowCount++;
logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EtlStepName = this.Name,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = e.Message,
Flags = e.RecordIndex,
StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
}
);
if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
{
result.Status = EtlStatus.FinishedWithLosses;
e.TrySkipError = true;
}
else
{
result.Status = EtlStatus.Failed;
}
};
insertedRowCount = dbWriter.Write(wrappedReader, this.TimeoutMilliseconds, this.BatchSize);
sourceRowCount = csvReader.CurrentRecordIndex + 1;
}
}
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Inserted",
// Flags = insertedRowCount,
//});
return(result);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
}
if (this.Destination == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
}
if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
}
if (string.IsNullOrEmpty(this.Destination.ConnectionString))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
}
if (string.IsNullOrEmpty(this.Destination.ProviderName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ProviderName"));
}
if (string.IsNullOrEmpty(this.Destination.TableName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.TableName"));
}
if (this.Source.FieldDelimiter.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
}
if (this.Source.Quote != null && this.Source.Quote.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
}
if (this.Source.Escape != null && this.Source.Escape.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var insertedRowCount = 0L;
var errorRowCount = 0L;
var csvSyntax = new CsvSyntaxInfo
{
HasHeaders = this.Source.HasHeaders,
FieldDelimiter = this.Source.FieldDelimiter[0],
Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
};
using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
{
using (var csvReader = new CsvReader(fileReader, csvSyntax))
{
var wrappedReader = new EtlMappedDataReader(csvReader, this.Mappings);
var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
{
var errorDateTime = DateTime.Now;
result.Status = EtlStatus.FinishedWithLosses;
result.Message = e.Message;
//todo: fix error counter in CSV import. Now it counts errors, not error records
errorRowCount++;
logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EtlStepName = this.Name,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = e.Message,
Flags = e.RecordIndex,
StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
}
);
if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
{
result.Status = EtlStatus.FinishedWithLosses;
e.TrySkipError = true;
}
else
{
result.Status = EtlStatus.Failed;
}
};
insertedRowCount = dbWriter.Write(wrappedReader, this.TimeoutMilliseconds, this.BatchSize);
sourceRowCount = csvReader.CurrentRecordIndex + 1;
}
}
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Inserted",
// Flags = insertedRowCount,
//});
return result;
}