public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException("ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException("ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProcedureName))
{
throw new InvalidOperationException("ProcedureName cannot be empty");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var hasOutputVariables = this.OutputVariables != null && this.OutputVariables.FirstRow.Count > 0;
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
if (hasOutputVariables)
{
using (var dbReader = dbAccessor.ExecuteProcedureReader(this.Source.ProcedureName, EtlProcedureParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
if (dbReader.Read())
{
foreach (var firstRowResult in this.OutputVariables.FirstRow)
{
var firstRowResultValue = EtlValueTranslation.Evaluate(firstRowResult.SourceFieldName, firstRowResult.SourceFieldTranslation, dbReader, firstRowResult.DefaultValue);
result.VariableAssignments.Add(new EtlVariableAssignment(firstRowResult.VariableName, EtlValueConverter.ToString(firstRowResultValue)));
}
}
}
}
else
{
dbAccessor.ExecuteProcedure(this.Source.ProcedureName, EtlProcedureParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds);
}
}
return result;
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException("ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException("ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProcedureName))
{
throw new InvalidOperationException("ProcedureName cannot be empty");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var hasOutputVariables = this.OutputVariables != null && this.OutputVariables.FirstRow.Count > 0;
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
if (hasOutputVariables)
{
using (var dbReader = dbAccessor.ExecuteProcedureReader(this.Source.ProcedureName, EtlProcedureParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
if (dbReader.Read())
{
foreach (var firstRowResult in this.OutputVariables.FirstRow)
{
var firstRowResultValue = EtlValueTranslation.Evaluate(firstRowResult.SourceFieldName, firstRowResult.SourceFieldTranslation, dbReader, firstRowResult.DefaultValue);
result.VariableAssignments.Add(new EtlVariableAssignment(firstRowResult.VariableName, EtlValueConverter.ToString(firstRowResultValue)));
}
}
}
}
else
{
dbAccessor.ExecuteProcedure(this.Source.ProcedureName, EtlProcedureParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds);
}
}
return(result);
}
public CsvValidator(EtlValidateCsvFileStep step, CsvReader reader, EtlContext context, IEtlLogger logger)
{
_step = step;
_reader = reader;
_context = context;
_logger = logger;
_reader.ParseError += OnReaderError;
}
public static void DoSomething(EtlContext context, IEtlLogger logger, string a, int b, DateTime c, string fileName)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
File.WriteAllText(fileName, string.Format("{0}, {1}, {2:o}", a, b, c));
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException("Source.FilePath cannot be empty");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Destination.Uri))
{
throw new InvalidOperationException("Destination.Uri cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.Method))
{
throw new InvalidOperationException("Destination.Method cannot be null");
}
if (!Directory.Exists(this.Source.FilePath))
{
throw new InvalidOperationException("Source directory should exist");
}
foreach (string file in Directory.GetFiles(this.Source.FilePath))
{
var webRequest = CreateRequest(Path.Combine(this.Destination.Uri, Path.GetFileName(file)));
UploadFile(webRequest, file);
}
return(new EtlStepResult(EtlStatus.Succeeded, null));
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException("Source.FilePath cannot be empty");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Destination.Uri))
{
throw new InvalidOperationException("Destination.Uri cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.Method))
{
throw new InvalidOperationException("Destination.Method cannot be null");
}
if (!Directory.Exists(this.Source.FilePath))
{
throw new InvalidOperationException("Source directory should exist");
}
foreach (string file in Directory.GetFiles(this.Source.FilePath))
{
var webRequest = CreateRequest(Path.Combine(this.Destination.Uri, Path.GetFileName(file)));
UploadFile(webRequest, file);
}
return new EtlStepResult(EtlStatus.Succeeded, null);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
return result;
}
private static object GetSpecialValue(ParameterInfo parameterInfo, EtlContext context, IEtlLogger logger)
{
if (parameterInfo.ParameterType == typeof(IEtlLogger))
{
return(logger);
}
else if (parameterInfo.ParameterType == typeof(EtlContext))
{
return(context);
}
else
{
return(null);
}
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
return(new EtlStepResult(EtlStatus.Failed, this.Message));
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException("Source.FilePath cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.FilePath))
{
throw new InvalidOperationException("Destination.FilePath cannot be empty");
}
var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);
if (!string.IsNullOrEmpty(destinationFolderPath))
{
if (!Directory.Exists(destinationFolderPath))
{
Directory.CreateDirectory(destinationFolderPath);
}
}
File.Copy(this.Source.FilePath, this.Destination.FilePath);
return(new EtlStepResult(EtlStatus.Succeeded, null));
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException("Source.FilePath cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.FilePath))
{
throw new InvalidOperationException("Destination.FilePath cannot be empty");
}
var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);
if (!string.IsNullOrEmpty(destinationFolderPath))
{
if (!Directory.Exists(destinationFolderPath))
{
Directory.CreateDirectory(destinationFolderPath);
}
}
File.Copy(this.Source.FilePath, this.Destination.FilePath);
return new EtlStepResult(EtlStatus.Succeeded, null);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.DelayMilliseconds > 0)
{
Thread.Sleep(this.DelayMilliseconds);
}
return(new EtlStepResult(EtlStatus.Succeeded, null));
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.DelayMilliseconds > 0)
{
Thread.Sleep(this.DelayMilliseconds);
}
return new EtlStepResult(EtlStatus.Succeeded, null);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException("Source.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException("Source.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.Text))
{
throw new InvalidOperationException("Source.Text cannot be empty");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var rowCounter = (this.Counters != null && this.Counters.RowCount != null && !string.IsNullOrEmpty(this.Counters.RowCount.CounterName))
? new EtlCounter
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EntityName = this.Counters.RowCount.EntityName,
CounterName = this.Counters.RowCount.CounterName,
}
: null;
var hasOutputVariables = this.OutputVariables != null && this.OutputVariables.FirstRow.Count > 0;
var hasRowCounter = rowCounter != null;
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
if (hasOutputVariables || hasRowCounter)
{
using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
if (dbReader.Read())
{
if (hasRowCounter)
{
rowCounter.CounterValue++;
}
if (hasOutputVariables)
{
foreach (var firstRowResult in this.OutputVariables.FirstRow)
{
var firstRowResultValue = EtlValueTranslation.Evaluate(firstRowResult.SourceFieldName, firstRowResult.SourceFieldTranslation, dbReader, firstRowResult.DefaultValue);
result.VariableAssignments.Add(new EtlVariableAssignment(firstRowResult.VariableName, EtlValueConverter.ToString(firstRowResultValue)));
}
}
}
if (hasRowCounter)
{
while (dbReader.Read())
{
rowCounter.CounterValue++;
}
rowCounter.DateTime = DateTime.Now;
rowCounter.UtcDateTime = rowCounter.DateTime.ToUniversalTime();
logger.LogEtlCounter(rowCounter);
}
}
}
else
{
dbAccessor.ExecuteNonQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds);
}
}
return(result);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException("Source.FilePath cannot be empty");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Destination.Uri))
{
throw new InvalidOperationException("Destination.Uri cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.Method))
{
throw new InvalidOperationException("Destination.Method cannot be null");
}
if (!File.Exists(this.Source.FilePath))
{
//throw exception
}
var webRequest = WebRequest.Create(this.Destination.Uri);
webRequest.Method = this.Destination.Method;
if (!string.IsNullOrEmpty(this.Destination.ContentType))
{
webRequest.ContentType = this.Destination.ContentType;
}
foreach (var header in this.Destination.Headers)
{
webRequest.Headers[header.Name] = header.Value;
}
if (this.TimeoutMilliseconds != null)
{
webRequest.Timeout = this.TimeoutMilliseconds.Value == 0 ? System.Threading.Timeout.Infinite : this.TimeoutMilliseconds.Value;
}
if (this.Destination.Credential != null && !string.IsNullOrEmpty(this.Destination.Credential.UserName))
{
var credential = new NetworkCredential(this.Destination.Credential.UserName, this.Destination.Credential.Password);
var credentialsCache = new CredentialCache();
credentialsCache.Add(new Uri(this.Destination.Uri), AUTHENTICATION_TYPE, credential);
webRequest.Credentials = credentialsCache;
}
if (this.Destination.AllowInvalidCertificates)
{
ServicePointManager.ServerCertificateValidationCallback +=
delegate
(
object sender,
X509Certificate certificate,
X509Chain chain,
SslPolicyErrors policyErrors
)
{
return(true);
};
}
byte[] fileBytes;
using (var sourceStream = new StreamReader(this.Source.FilePath))
{
var encoding = Encoding.GetEncoding(this.Source.CodePage);
fileBytes = encoding.GetBytes(sourceStream.ReadToEnd());
sourceStream.Close();
}
webRequest.ContentLength = fileBytes.Length;
var requestStream = webRequest.GetRequestStream();
requestStream.Write(fileBytes, 0, fileBytes.Length);
requestStream.Close();
return(new EtlStepResult(EtlStatus.Succeeded, null));
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException("Source.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException("Source.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.Text))
{
throw new InvalidOperationException("Source.Text cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.ConnectionString))
{
throw new InvalidOperationException("Destination.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.ProviderName))
{
throw new InvalidOperationException("Destination.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.TableName))
{
throw new InvalidOperationException("Destination.TableName cannot be empty");
}
if (this.Mappings.Count == 0)
{
throw new InvalidOperationException("Mappings not specified");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var insertedRowCount = 0L;
var errorRowCount = 0L;
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
{
var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
{
var errorDateTime = DateTime.Now;
result.Status = EtlStatus.FinishedWithLosses;
result.Message = e.Message;
errorRowCount++;
logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EtlStepName = this.Name,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = e.Message,
Flags = e.RecordIndex,
StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
}
);
if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
{
result.Status = EtlStatus.FinishedWithLosses;
e.TrySkipError = true;
}
else
{
result.Status = EtlStatus.Failed;
}
};
insertedRowCount = dbWriter.Write(mapReader, this.TimeoutMilliseconds, this.BatchSize);
}
}
}
sourceRowCount = insertedRowCount + errorRowCount;
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Found",
// Flags = sourceRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Inserted",
// Flags = insertedRowCount,
//});
return(result);
}
private static object GetSpecialValue(ParameterInfo parameterInfo, EtlContext context, IEtlLogger logger)
{
if (parameterInfo.ParameterType == typeof(IEtlLogger))
{
return logger;
}
else if (parameterInfo.ParameterType == typeof(EtlContext))
{
return context;
}
else
{
return null;
}
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
}
if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
}
if (this.Source.FieldDelimiter.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
}
if (this.Source.Quote != null && this.Source.Quote.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
}
if (this.Source.Escape != null && this.Source.Escape.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var validRowCount = 0L;
var errorRowCount = 0L;
var csvSyntax = new CsvSyntaxInfo
{
HasHeaders = this.Source.HasHeaders,
FieldDelimiter = this.Source.FieldDelimiter[0],
Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
};
using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
{
using (var csvReader = new CsvReader(fileReader, csvSyntax))
{
var validator = new CsvValidator(this, csvReader, context, logger);
result.Status = validator.Validate();
sourceRowCount = validator.ReadRowCount;
validRowCount = validator.ValidRowCount;
errorRowCount = validator.ErrorRowCount;
}
}
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Found",
// Flags = sourceRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Validated",
// Flags = validRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
return(result);
}
public static ExecutionResult Execute(CommandContext commandContext, CompiledModule module, string[] commands)
{
var result = ExecutionResult.Success;
var etlContext = new EtlContext();
try
{
var listeners = commandContext.HostConfiguration.GetSessionListeners(null);
if (listeners?.Count > 0)
{
etlContext.Listeners.AddRange(listeners);
}
}
catch (Exception ex)
{
var formattedMessage = ex.FormatExceptionWithDetails();
etlContext.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
etlContext.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
}
var instance = Environment.MachineName;
var arguments = GetArguments(module, instance);
var environmentSettings = new EnvironmentSettings();
module.Startup.Configure(environmentSettings);
var taskCreators = new Dictionary <string, Func <IEtlSessionArguments, IEtlTask> >(module.Startup.Commands, StringComparer.InvariantCultureIgnoreCase);
var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
var session = new EtlSession(sessionId, etlContext, arguments);
etlContext.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;
var logger = CreateLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var opsLogger = CreateOpsLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var ioLogger = CreateIoLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var serilogAdapter = new EtlSessionSerilogAdapter(logger, opsLogger, ioLogger, commandContext.DevLogFolder, commandContext.OpsLogFolder);
etlContext.Listeners.Add(serilogAdapter);
serilogAdapter.Log(LogSeverity.Information, false, null, null, "session {SessionId} started", sessionId);
if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
etlContext.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
}
var sessionStartedOn = Stopwatch.StartNew();
var sessionExceptionCount = 0;
var taskResults = new List <KeyValuePair <IEtlTask, TaskResult> >();
try
{
foreach (var command in commands)
{
IEtlTask task = null;
if (taskCreators.TryGetValue(command, out var taskCreator))
{
task = taskCreator.Invoke(arguments);
}
else
{
var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, command, StringComparison.InvariantCultureIgnoreCase));
if (taskType != null)
{
task = (IEtlTask)Activator.CreateInstance(taskType);
}
}
if (task == null)
{
serilogAdapter.Log(LogSeverity.Error, false, null, null, "unknown task/flow type: " + command);
break;
}
try
{
try
{
var taskResult = session.ExecuteTask(null, task);
taskResults.Add(new KeyValuePair <IEtlTask, TaskResult>(task, taskResult));
sessionExceptionCount += taskResult.ExceptionCount;
if (sessionExceptionCount > 0)
{
etlContext.Log(LogSeverity.Error, task, "failed, terminating execution");
result = ExecutionResult.ExecutionFailed;
etlContext.Close();
break; // stop processing tasks
}
}
catch (Exception ex)
{
etlContext.Log(LogSeverity.Error, task, "failed, terminating execution, reason: ", task.Statistics.RunTime, ex.Message);
result = ExecutionResult.ExecutionFailed;
etlContext.Close();
break; // stop processing tasks
}
}
catch (TransactionAbortedException)
{
}
LogTaskCounters(serilogAdapter, task);
}
session.Stop();
if (taskResults.Count > 0)
{
serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
serilogAdapter.Log(LogSeverity.Information, false, null, null, "SUMMARY");
serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
var longestTaskName = taskResults.Max(x => x.Key.Name.Length);
foreach (var kvp in taskResults)
{
LogTaskSummary(serilogAdapter, kvp.Key, kvp.Value, longestTaskName);
}
}
etlContext.Close();
}
catch (TransactionAbortedException)
{
}
return(result);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
}
if (this.Destination == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.ProviderName"));
}
if (string.IsNullOrEmpty(this.Source.Text))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.Text"));
}
if (string.IsNullOrEmpty(this.Destination.FieldDelimiter))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FieldDelimiter"));
}
if (string.IsNullOrEmpty(this.Destination.FilePath))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FilePath"));
}
if (this.Destination.FieldDelimiter.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.FieldDelimiter", 1));
}
if (this.Destination.Quote != null && this.Destination.Quote.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Quote", 1));
}
if (this.Destination.Escape != null && this.Destination.Escape.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Escape", 1));
}
if (this.Destination != null)
{
var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);
if (!string.IsNullOrEmpty(destinationFolderPath))
{
if (!Directory.Exists(destinationFolderPath))
{
Directory.CreateDirectory(destinationFolderPath);
}
}
}
//if (this.Mappings.Count == 0)
//{
// throw new InvalidOperationException("Mappings not specified");
//}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var writtenRowCount = 0L;
var encoding = Encoding.GetEncoding(this.Destination.CodePage);
var csvSyntax = new CsvSyntaxInfo
{
HasHeaders = this.Destination.HasHeaders,
FieldDelimiter = this.Destination.FieldDelimiter[0],
Quote = this.Destination.Quote != null ? this.Destination.Quote[0] : DefaultQuote,
Escape = this.Destination.Escape != null ? this.Destination.Escape[0] : DefaultEscape,
LineDelimiter1 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[0] : DefaultLineDelimiter[0],
LineDelimiter2 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[1] : DefaultLineDelimiter[1],
};
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
{
using (var fileWriter = new StreamWriter(this.Destination.FilePath, false, encoding))
{
var csvWriter = new CsvWriter(fileWriter, csvSyntax);
writtenRowCount = csvWriter.Write(mapReader);
sourceRowCount = writtenRowCount;
}
}
}
}
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Found",
// Flags = sourceRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Exported",
// Flags = writtenRowCount,
//});
return result;
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.Uri))
{
throw new InvalidOperationException("Source.Uri cannot be empty");
}
if (this.Destination != null && string.IsNullOrEmpty(this.Destination.FilePath))
{
throw new InvalidOperationException("Destination.FilePath cannot be empty");
}
if (this.Destination != null)
{
var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);
if (!string.IsNullOrEmpty(destinationFolderPath))
{
if (!Directory.Exists(destinationFolderPath))
{
Directory.CreateDirectory(destinationFolderPath);
}
}
}
//получение списка файлов
var listDirWebRequest = GetRequest(WebRequestMethods.Ftp.ListDirectory, this.Source.Uri);
if (this.Source.AllowInvalidCertificates)
{
ServicePointManager.ServerCertificateValidationCallback +=
delegate
(
object sender,
X509Certificate certificate,
X509Chain chain,
SslPolicyErrors policyErrors
)
{
return true;
};
}
if (!string.IsNullOrEmpty(this.Source.Request))
{
var bytes = Encoding.UTF8.GetBytes(this.Source.Request);
listDirWebRequest.ContentLength = bytes.Length;
var request = listDirWebRequest.GetRequestStream();
request.Write(bytes, 0, bytes.Length);
request.Close();
}
var files = GetFileList(listDirWebRequest);
//получение файлов
foreach (var file in files)
{
var fileRequest = GetRequest(WebRequestMethods.Ftp.DownloadFile, this.Source.Uri+"/"+file);
using (var response = fileRequest.GetResponse())
{
if (this.Destination != null)
{
using (var responseStream = response.GetResponseStream())
{
using (var fileStream = File.Create(this.Destination.FilePath+"\\"+file))
{
var buffer = new byte[1024];
var bytesRead = 0;
//var bytesProcessed = 0;
do
{
bytesRead = responseStream.Read(buffer, 0, buffer.Length);
fileStream.Write(buffer, 0, bytesRead);
//bytesProcessed += bytesRead;
} while (bytesRead > 0);
}
}
}
}
}
return new EtlStepResult(EtlStatus.Succeeded, null);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.AssemblyName))
{
throw new InvalidOperationException("Source.Assembly cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.TypeName))
{
throw new InvalidOperationException("Source.TypeName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.MethodName))
{
throw new InvalidOperationException("Source.MethodName cannot be empty");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourcePath = string.Format("{0}, {1}, {2}", this.Source.AssemblyName, this.Source.TypeName, this.Source.MethodName);
var asm = Assembly.Load(this.Source.AssemblyName);
var type = asm.GetType(this.Source.TypeName);
if (type == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.InvokeMethodCannotFindType, this.Source.AssemblyName, this.Source.TypeName));
}
var method = type.GetMethod(this.Source.MethodName);
if (method == null)
{
throw new InvalidOperationException("Method '" + Source.MethodName + "' does not exist within assembly " + Source.AssemblyName);
}
if (!method.IsStatic)
{
throw new InvalidOperationException(string.Format(Properties.Resources.InvokeMethodCannotInvokeNonStaticMethod, this.Source.AssemblyName, this.Source.TypeName, this.Source.MethodName));
}
var values = GetParameterValues(method, this.Source.Parameters, context, logger);
var res = method.Invoke(null, values);
if (res is EtlStepResult)
{
result = (EtlStepResult)res;
}
return(result);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.Uri))
{
throw new InvalidOperationException("Source.Uri cannot be empty");
}
if (this.Destination != null && string.IsNullOrEmpty(this.Destination.FilePath))
{
throw new InvalidOperationException("Destination.FilePath cannot be empty");
}
if (this.Destination != null)
{
var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);
if (!string.IsNullOrEmpty(destinationFolderPath))
{
if (!Directory.Exists(destinationFolderPath))
{
Directory.CreateDirectory(destinationFolderPath);
}
}
}
//получение списка файлов
var listDirWebRequest = GetRequest(WebRequestMethods.Ftp.ListDirectory, this.Source.Uri);
if (this.Source.AllowInvalidCertificates)
{
ServicePointManager.ServerCertificateValidationCallback +=
delegate
(
object sender,
X509Certificate certificate,
X509Chain chain,
SslPolicyErrors policyErrors
)
{
return(true);
};
}
if (!string.IsNullOrEmpty(this.Source.Request))
{
var bytes = Encoding.UTF8.GetBytes(this.Source.Request);
listDirWebRequest.ContentLength = bytes.Length;
var request = listDirWebRequest.GetRequestStream();
request.Write(bytes, 0, bytes.Length);
request.Close();
}
var files = GetFileList(listDirWebRequest);
//получение файлов
foreach (var file in files)
{
var fileRequest = GetRequest(WebRequestMethods.Ftp.DownloadFile, this.Source.Uri + "/" + file);
using (var response = fileRequest.GetResponse())
{
if (this.Destination != null)
{
using (var responseStream = response.GetResponseStream())
{
using (var fileStream = File.Create(this.Destination.FilePath + "\\" + file))
{
var buffer = new byte[1024];
var bytesRead = 0;
//var bytesProcessed = 0;
do
{
bytesRead = responseStream.Read(buffer, 0, buffer.Length);
fileStream.Write(buffer, 0, bytesRead);
//bytesProcessed += bytesRead;
} while (bytesRead > 0);
}
}
}
}
}
return(new EtlStepResult(EtlStatus.Succeeded, null));
}
private static object[] GetParameterValues(MethodInfo method, IEnumerable<EtlMethodParameter> parameters, EtlContext context, IEtlLogger logger)
{
var parametersInfo = method.GetParameters();
var values = new object[parametersInfo.Length];
for (var i = 0; i < parametersInfo.Length; i++)
{
var parameterInfo = parametersInfo[i];
var parameter = FindParameter(parameters, parameterInfo.Name);
if (parameter == null)
{
values[i] = GetSpecialValue(parameterInfo, context, logger);
}
else
{
values[i] = EtlValueConverter.ParseType(parameter.Value, Type.GetTypeCode(parameterInfo.ParameterType));
}
}
return values;
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
}
if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
}
if (this.Source.FieldDelimiter.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
}
if (this.Source.Quote != null && this.Source.Quote.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
}
if (this.Source.Escape != null && this.Source.Escape.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var validRowCount = 0L;
var errorRowCount = 0L;
var csvSyntax = new CsvSyntaxInfo
{
HasHeaders = this.Source.HasHeaders,
FieldDelimiter = this.Source.FieldDelimiter[0],
Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
};
using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
{
using (var csvReader = new CsvReader(fileReader, csvSyntax))
{
var validator = new CsvValidator(this, csvReader, context, logger);
result.Status = validator.Validate();
sourceRowCount = validator.ReadRowCount;
validRowCount = validator.ValidRowCount;
errorRowCount = validator.ErrorRowCount;
}
}
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Found",
// Flags = sourceRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Validated",
// Flags = validRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
return result;
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.AssemblyName))
{
throw new InvalidOperationException("Source.Assembly cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.TypeName))
{
throw new InvalidOperationException("Source.TypeName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.MethodName))
{
throw new InvalidOperationException("Source.MethodName cannot be empty");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourcePath = string.Format("{0}, {1}, {2}", this.Source.AssemblyName, this.Source.TypeName, this.Source.MethodName);
var asm = Assembly.Load(this.Source.AssemblyName);
var type = asm.GetType(this.Source.TypeName);
if (type == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.InvokeMethodCannotFindType, this.Source.AssemblyName, this.Source.TypeName));
}
var method = type.GetMethod(this.Source.MethodName);
if (method == null)
{
throw new InvalidOperationException("Method '" + Source.MethodName + "' does not exist within assembly " + Source.AssemblyName);
}
if (!method.IsStatic)
{
throw new InvalidOperationException(string.Format(Properties.Resources.InvokeMethodCannotInvokeNonStaticMethod, this.Source.AssemblyName, this.Source.TypeName, this.Source.MethodName));
}
var values = GetParameterValues(method, this.Source.Parameters, context, logger);
var res = method.Invoke(null, values);
if (res is EtlStepResult)
{
result = (EtlStepResult) res;
}
return result;
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
}
if (this.Destination == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
}
if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
}
if (string.IsNullOrEmpty(this.Destination.ConnectionString))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
}
if (string.IsNullOrEmpty(this.Destination.ProviderName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ProviderName"));
}
if (string.IsNullOrEmpty(this.Destination.TableName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.TableName"));
}
if (this.Source.FieldDelimiter.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
}
if (this.Source.Quote != null && this.Source.Quote.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
}
if (this.Source.Escape != null && this.Source.Escape.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var insertedRowCount = 0L;
var errorRowCount = 0L;
var csvSyntax = new CsvSyntaxInfo
{
HasHeaders = this.Source.HasHeaders,
FieldDelimiter = this.Source.FieldDelimiter[0],
Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
};
using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
{
using (var csvReader = new CsvReader(fileReader, csvSyntax))
{
var wrappedReader = new EtlMappedDataReader(csvReader, this.Mappings);
var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
{
var errorDateTime = DateTime.Now;
result.Status = EtlStatus.FinishedWithLosses;
result.Message = e.Message;
//todo: fix error counter in CSV import. Now it counts errors, not error records
errorRowCount++;
logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EtlStepName = this.Name,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = e.Message,
Flags = e.RecordIndex,
StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
}
);
if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
{
result.Status = EtlStatus.FinishedWithLosses;
e.TrySkipError = true;
}
else
{
result.Status = EtlStatus.Failed;
}
};
insertedRowCount = dbWriter.Write(wrappedReader, this.TimeoutMilliseconds, this.BatchSize);
sourceRowCount = csvReader.CurrentRecordIndex + 1;
}
}
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Inserted",
// Flags = insertedRowCount,
//});
return result;
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException("Source.FilePath cannot be empty");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Destination.Uri))
{
throw new InvalidOperationException("Destination.Uri cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.Method))
{
throw new InvalidOperationException("Destination.Method cannot be null");
}
if (!File.Exists(this.Source.FilePath))
{
//throw exception
}
var webRequest = WebRequest.Create(this.Destination.Uri);
webRequest.Method = this.Destination.Method;
if (!string.IsNullOrEmpty(this.Destination.ContentType))
{
webRequest.ContentType = this.Destination.ContentType;
}
foreach (var header in this.Destination.Headers)
{
webRequest.Headers[header.Name] = header.Value;
}
if (this.TimeoutMilliseconds != null)
{
webRequest.Timeout = this.TimeoutMilliseconds.Value == 0 ? System.Threading.Timeout.Infinite : this.TimeoutMilliseconds.Value;
}
if (this.Destination.Credential != null && !string.IsNullOrEmpty(this.Destination.Credential.UserName))
{
var credential = new NetworkCredential(this.Destination.Credential.UserName, this.Destination.Credential.Password);
var credentialsCache = new CredentialCache();
credentialsCache.Add(new Uri(this.Destination.Uri), AUTHENTICATION_TYPE, credential);
webRequest.Credentials = credentialsCache;
}
if (this.Destination.AllowInvalidCertificates)
{
ServicePointManager.ServerCertificateValidationCallback +=
delegate
(
object sender,
X509Certificate certificate,
X509Chain chain,
SslPolicyErrors policyErrors
)
{
return true;
};
}
byte[] fileBytes;
using (var sourceStream = new StreamReader(this.Source.FilePath))
{
var encoding = Encoding.GetEncoding(this.Source.CodePage);
fileBytes = encoding.GetBytes(sourceStream.ReadToEnd());
sourceStream.Close();
}
webRequest.ContentLength = fileBytes.Length;
var requestStream = webRequest.GetRequestStream();
requestStream.Write(fileBytes, 0, fileBytes.Length);
requestStream.Close();
return new EtlStepResult(EtlStatus.Succeeded, null);
}
private static object[] GetParameterValues(MethodInfo method, IEnumerable <EtlMethodParameter> parameters, EtlContext context, IEtlLogger logger)
{
var parametersInfo = method.GetParameters();
var values = new object[parametersInfo.Length];
for (var i = 0; i < parametersInfo.Length; i++)
{
var parameterInfo = parametersInfo[i];
var parameter = FindParameter(parameters, parameterInfo.Name);
if (parameter == null)
{
values[i] = GetSpecialValue(parameterInfo, context, logger);
}
else
{
values[i] = EtlValueConverter.ParseType(parameter.Value, Type.GetTypeCode(parameterInfo.ParameterType));
}
}
return(values);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.Uri))
{
throw new InvalidOperationException("Source.Uri cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.Method))
{
throw new InvalidOperationException("Source.Method cannot be null");
}
if (this.Destination != null && string.IsNullOrEmpty(this.Destination.FilePath))
{
throw new InvalidOperationException("Destination.FilePath cannot be empty");
}
if (this.Destination != null)
{
var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);
if (!string.IsNullOrEmpty(destinationFolderPath))
{
if (!Directory.Exists(destinationFolderPath))
{
Directory.CreateDirectory(destinationFolderPath);
}
}
}
var webRequest = WebRequest.Create(this.Source.Uri);
webRequest.Method = this.Source.Method;
if (!string.IsNullOrEmpty(this.Source.ContentType))
{
webRequest.ContentType = this.Source.ContentType;
}
foreach (var header in this.Source.Headers)
{
webRequest.Headers[header.Name] = header.Value;
}
if (this.TimeoutMilliseconds != null)
{
webRequest.Timeout = this.TimeoutMilliseconds.Value == 0 ? System.Threading.Timeout.Infinite : this.TimeoutMilliseconds.Value;
}
if (this.Source.Credential != null && !string.IsNullOrEmpty(this.Source.Credential.UserName))
{
var credential = new NetworkCredential(this.Source.Credential.UserName, this.Source.Credential.Password);
var credentialsCache = new CredentialCache();
credentialsCache.Add(new Uri(this.Source.Uri), AUTHENTICATION_TYPE, credential);
webRequest.Credentials = credentialsCache;
}
if (this.Source.AllowInvalidCertificates)
{
ServicePointManager.ServerCertificateValidationCallback +=
delegate
(
object sender,
X509Certificate certificate,
X509Chain chain,
SslPolicyErrors policyErrors
)
{
return(true);
};
}
if (!string.IsNullOrEmpty(this.Source.Request))
{
var bytes = Encoding.UTF8.GetBytes(this.Source.Request);
webRequest.ContentLength = bytes.Length;
var request = webRequest.GetRequestStream();
request.Write(bytes, 0, bytes.Length);
request.Close();
}
using (var response = webRequest.GetResponse())
{
if (this.Destination != null)
{
using (var responseStream = response.GetResponseStream())
{
using (var fileStream = File.Create(this.Destination.FilePath))
{
var buffer = new byte[1024];
var bytesRead = 0;
//var bytesProcessed = 0;
do
{
bytesRead = responseStream.Read(buffer, 0, buffer.Length);
fileStream.Write(buffer, 0, bytesRead);
//bytesProcessed += bytesRead;
} while (bytesRead > 0);
}
}
}
}
return(new EtlStepResult(EtlStatus.Succeeded, null));
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
}
if (this.Destination == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.ProviderName"));
}
if (string.IsNullOrEmpty(this.Source.Text))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.Text"));
}
if (string.IsNullOrEmpty(this.Destination.FieldDelimiter))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FieldDelimiter"));
}
if (string.IsNullOrEmpty(this.Destination.FilePath))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FilePath"));
}
if (this.Destination.FieldDelimiter.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.FieldDelimiter", 1));
}
if (this.Destination.Quote != null && this.Destination.Quote.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Quote", 1));
}
if (this.Destination.Escape != null && this.Destination.Escape.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Escape", 1));
}
if (this.Destination != null)
{
var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);
if (!string.IsNullOrEmpty(destinationFolderPath))
{
if (!Directory.Exists(destinationFolderPath))
{
Directory.CreateDirectory(destinationFolderPath);
}
}
}
//if (this.Mappings.Count == 0)
//{
// throw new InvalidOperationException("Mappings not specified");
//}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var writtenRowCount = 0L;
var encoding = Encoding.GetEncoding(this.Destination.CodePage);
var csvSyntax = new CsvSyntaxInfo
{
HasHeaders = this.Destination.HasHeaders,
FieldDelimiter = this.Destination.FieldDelimiter[0],
Quote = this.Destination.Quote != null ? this.Destination.Quote[0] : DefaultQuote,
Escape = this.Destination.Escape != null ? this.Destination.Escape[0] : DefaultEscape,
LineDelimiter1 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[0] : DefaultLineDelimiter[0],
LineDelimiter2 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[1] : DefaultLineDelimiter[1],
};
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
{
using (var fileWriter = new StreamWriter(this.Destination.FilePath, false, encoding))
{
var csvWriter = new CsvWriter(fileWriter, csvSyntax);
writtenRowCount = csvWriter.Write(mapReader);
sourceRowCount = writtenRowCount;
}
}
}
}
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Found",
// Flags = sourceRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Exported",
// Flags = writtenRowCount,
//});
return(result);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
}
if (this.Destination == null)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
}
if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
}
if (string.IsNullOrEmpty(this.Source.FilePath))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
}
if (string.IsNullOrEmpty(this.Destination.ConnectionString))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
}
if (string.IsNullOrEmpty(this.Destination.ProviderName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ProviderName"));
}
if (string.IsNullOrEmpty(this.Destination.TableName))
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.TableName"));
}
if (this.Source.FieldDelimiter.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
}
if (this.Source.Quote != null && this.Source.Quote.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
}
if (this.Source.Escape != null && this.Source.Escape.Length != 1)
{
throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var insertedRowCount = 0L;
var errorRowCount = 0L;
var csvSyntax = new CsvSyntaxInfo
{
HasHeaders = this.Source.HasHeaders,
FieldDelimiter = this.Source.FieldDelimiter[0],
Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
};
using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
{
using (var csvReader = new CsvReader(fileReader, csvSyntax))
{
var wrappedReader = new EtlMappedDataReader(csvReader, this.Mappings);
var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
{
var errorDateTime = DateTime.Now;
result.Status = EtlStatus.FinishedWithLosses;
result.Message = e.Message;
//todo: fix error counter in CSV import. Now it counts errors, not error records
errorRowCount++;
logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EtlStepName = this.Name,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = e.Message,
Flags = e.RecordIndex,
StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
}
);
if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
{
result.Status = EtlStatus.FinishedWithLosses;
e.TrySkipError = true;
}
else
{
result.Status = EtlStatus.Failed;
}
};
insertedRowCount = dbWriter.Write(wrappedReader, this.TimeoutMilliseconds, this.BatchSize);
sourceRowCount = csvReader.CurrentRecordIndex + 1;
}
}
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Inserted",
// Flags = insertedRowCount,
//});
return(result);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (this.Destination == null)
{
throw new InvalidOperationException("Destination cannot be null");
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException("Source.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException("Source.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.Text))
{
throw new InvalidOperationException("Source.Text cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.ConnectionString))
{
throw new InvalidOperationException("Destination.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.ProviderName))
{
throw new InvalidOperationException("Destination.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Destination.TableName))
{
throw new InvalidOperationException("Destination.TableName cannot be empty");
}
if (this.Mappings.Count == 0)
{
throw new InvalidOperationException("Mappings not specified");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var sourceRowCount = 0L;
var insertedRowCount = 0L;
var errorRowCount = 0L;
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
{
var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
{
var errorDateTime = DateTime.Now;
result.Status = EtlStatus.FinishedWithLosses;
result.Message = e.Message;
errorRowCount++;
logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EtlStepName = this.Name,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = e.Message,
Flags = e.RecordIndex,
StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
}
);
if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
{
result.Status = EtlStatus.FinishedWithLosses;
e.TrySkipError = true;
}
else
{
result.Status = EtlStatus.Failed;
}
};
insertedRowCount = dbWriter.Write(mapReader, this.TimeoutMilliseconds, this.BatchSize);
}
}
}
sourceRowCount = insertedRowCount + errorRowCount;
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Found",
// Flags = sourceRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Errors",
// Flags = errorRowCount,
//});
//logger.LogEtlMessage(new EtlMessage
//{
// EtlPackageId = context.EtlPackageId,
// EtlSessionId = context.EtlSessionId,
// EtlStepId = this.Id,
// LogDateTime = endDateTime,
// LogUtcDateTime = endDateTime.ToUniversalTime(),
// MessageType = EtlMessageType.Statistics,
// Text = "Inserted",
// Flags = insertedRowCount,
//});
return result;
}
public CsvValidator(EtlValidateCsvFileStep step, CsvReader reader, EtlContext context, IEtlLogger logger)
{
_step = step;
_reader = reader;
_context = context;
_logger = logger;
_reader.ParseError += OnReaderError;
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.Uri))
{
throw new InvalidOperationException("Source.Uri cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.Method))
{
throw new InvalidOperationException("Source.Method cannot be null");
}
if (this.Destination != null && string.IsNullOrEmpty(this.Destination.FilePath))
{
throw new InvalidOperationException("Destination.FilePath cannot be empty");
}
if (this.Destination != null)
{
var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);
if (!string.IsNullOrEmpty(destinationFolderPath))
{
if (!Directory.Exists(destinationFolderPath))
{
Directory.CreateDirectory(destinationFolderPath);
}
}
}
var webRequest = WebRequest.Create(this.Source.Uri);
webRequest.Method = this.Source.Method;
if (!string.IsNullOrEmpty(this.Source.ContentType))
{
webRequest.ContentType = this.Source.ContentType;
}
foreach (var header in this.Source.Headers)
{
webRequest.Headers[header.Name] = header.Value;
}
if (this.TimeoutMilliseconds != null)
{
webRequest.Timeout = this.TimeoutMilliseconds.Value == 0 ? System.Threading.Timeout.Infinite : this.TimeoutMilliseconds.Value;
}
if (this.Source.Credential != null && !string.IsNullOrEmpty(this.Source.Credential.UserName))
{
var credential = new NetworkCredential(this.Source.Credential.UserName, this.Source.Credential.Password);
var credentialsCache = new CredentialCache();
credentialsCache.Add(new Uri(this.Source.Uri), AUTHENTICATION_TYPE, credential);
webRequest.Credentials = credentialsCache;
}
if (this.Source.AllowInvalidCertificates)
{
ServicePointManager.ServerCertificateValidationCallback +=
delegate
(
object sender,
X509Certificate certificate,
X509Chain chain,
SslPolicyErrors policyErrors
)
{
return true;
};
}
if (!string.IsNullOrEmpty(this.Source.Request))
{
var bytes = Encoding.UTF8.GetBytes(this.Source.Request);
webRequest.ContentLength = bytes.Length;
var request = webRequest.GetRequestStream();
request.Write(bytes, 0, bytes.Length);
request.Close();
}
using (var response = webRequest.GetResponse())
{
if (this.Destination != null)
{
using (var responseStream = response.GetResponseStream())
{
using (var fileStream = File.Create(this.Destination.FilePath))
{
var buffer = new byte[1024];
var bytesRead = 0;
//var bytesProcessed = 0;
do
{
bytesRead = responseStream.Read(buffer, 0, buffer.Length);
fileStream.Write(buffer, 0, bytesRead);
//bytesProcessed += bytesRead;
} while (bytesRead > 0);
}
}
}
}
return new EtlStepResult(EtlStatus.Succeeded, null);
}
public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (this.Source == null)
{
throw new InvalidOperationException("Source cannot be null");
}
if (string.IsNullOrEmpty(this.Source.ConnectionString))
{
throw new InvalidOperationException("Source.ConnectionString cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.ProviderName))
{
throw new InvalidOperationException("Source.ProviderName cannot be empty");
}
if (string.IsNullOrEmpty(this.Source.Text))
{
throw new InvalidOperationException("Source.Text cannot be empty");
}
var result = new EtlStepResult(EtlStatus.Succeeded, null);
var rowCounter = (this.Counters != null && this.Counters.RowCount != null && !string.IsNullOrEmpty(this.Counters.RowCount.CounterName))
? new EtlCounter
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
EntityName = this.Counters.RowCount.EntityName,
CounterName = this.Counters.RowCount.CounterName,
}
: null;
var hasOutputVariables = this.OutputVariables != null && this.OutputVariables.FirstRow.Count > 0;
var hasRowCounter = rowCounter != null;
using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
{
if (hasOutputVariables || hasRowCounter)
{
using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
{
if (dbReader.Read())
{
if (hasRowCounter)
{
rowCounter.CounterValue++;
}
if (hasOutputVariables)
{
foreach (var firstRowResult in this.OutputVariables.FirstRow)
{
var firstRowResultValue = EtlValueTranslation.Evaluate(firstRowResult.SourceFieldName, firstRowResult.SourceFieldTranslation, dbReader, firstRowResult.DefaultValue);
result.VariableAssignments.Add(new EtlVariableAssignment(firstRowResult.VariableName, EtlValueConverter.ToString(firstRowResultValue)));
}
}
}
if (hasRowCounter)
{
while (dbReader.Read())
{
rowCounter.CounterValue++;
}
rowCounter.DateTime = DateTime.Now;
rowCounter.UtcDateTime = rowCounter.DateTime.ToUniversalTime();
logger.LogEtlCounter(rowCounter);
}
}
}
else
{
dbAccessor.ExecuteNonQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds);
}
}
return result;
}
public static void DoSomething(EtlContext context, IEtlLogger logger, string a, int b, DateTime c, string fileName)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
File.WriteAllText(fileName, string.Format("{0}, {1}, {2:o}", a, b, c));
}