public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.ConnectionString))
            {
                throw new InvalidOperationException("ConnectionString cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.ProviderName))
            {
                throw new InvalidOperationException("ProviderName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.ProcedureName))
            {
                throw new InvalidOperationException("ProcedureName cannot be empty");
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);
            var hasOutputVariables = this.OutputVariables != null && this.OutputVariables.FirstRow.Count > 0;

            using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
            {
                if (hasOutputVariables)
                {
                    using (var dbReader = dbAccessor.ExecuteProcedureReader(this.Source.ProcedureName, EtlProcedureParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
                    {
                        if (dbReader.Read())
                        {
                            foreach (var firstRowResult in this.OutputVariables.FirstRow)
                            {
                                var firstRowResultValue = EtlValueTranslation.Evaluate(firstRowResult.SourceFieldName, firstRowResult.SourceFieldTranslation, dbReader, firstRowResult.DefaultValue);
                                result.VariableAssignments.Add(new EtlVariableAssignment(firstRowResult.VariableName, EtlValueConverter.ToString(firstRowResultValue)));
                            }
                        }
                    }
                }
                else
                {
                    dbAccessor.ExecuteProcedure(this.Source.ProcedureName, EtlProcedureParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds);
                }
            }

            return result;
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.ConnectionString))
            {
                throw new InvalidOperationException("ConnectionString cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.ProviderName))
            {
                throw new InvalidOperationException("ProviderName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.ProcedureName))
            {
                throw new InvalidOperationException("ProcedureName cannot be empty");
            }

            var result             = new EtlStepResult(EtlStatus.Succeeded, null);
            var hasOutputVariables = this.OutputVariables != null && this.OutputVariables.FirstRow.Count > 0;

            using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
            {
                if (hasOutputVariables)
                {
                    using (var dbReader = dbAccessor.ExecuteProcedureReader(this.Source.ProcedureName, EtlProcedureParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
                    {
                        if (dbReader.Read())
                        {
                            foreach (var firstRowResult in this.OutputVariables.FirstRow)
                            {
                                var firstRowResultValue = EtlValueTranslation.Evaluate(firstRowResult.SourceFieldName, firstRowResult.SourceFieldTranslation, dbReader, firstRowResult.DefaultValue);
                                result.VariableAssignments.Add(new EtlVariableAssignment(firstRowResult.VariableName, EtlValueConverter.ToString(firstRowResultValue)));
                            }
                        }
                    }
                }
                else
                {
                    dbAccessor.ExecuteProcedure(this.Source.ProcedureName, EtlProcedureParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds);
                }
            }

            return(result);
        }
Example #3
0
            public CsvValidator(EtlValidateCsvFileStep step, CsvReader reader, EtlContext context, IEtlLogger logger)
            {
                _step    = step;
                _reader  = reader;
                _context = context;
                _logger  = logger;

                _reader.ParseError += OnReaderError;
            }
        public static void DoSomething(EtlContext context, IEtlLogger logger, string a, int b, DateTime c, string fileName)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            File.WriteAllText(fileName, string.Format("{0}, {1}, {2:o}", a, b, c));
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException("Source.FilePath cannot be empty");
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException("Destination cannot be null");
            }

            if (string.IsNullOrEmpty(this.Destination.Uri))
            {
                throw new InvalidOperationException("Destination.Uri cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.Method))
            {
                throw new InvalidOperationException("Destination.Method cannot be null");
            }

            if (!Directory.Exists(this.Source.FilePath))
            {
                throw new InvalidOperationException("Source directory should exist");
            }

            foreach (string file in Directory.GetFiles(this.Source.FilePath))
            {
                var webRequest = CreateRequest(Path.Combine(this.Destination.Uri, Path.GetFileName(file)));
                UploadFile(webRequest, file);
            }
            return(new EtlStepResult(EtlStatus.Succeeded, null));
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException("Source.FilePath cannot be empty");
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException("Destination cannot be null");
            }

            if (string.IsNullOrEmpty(this.Destination.Uri))
            {
                throw new InvalidOperationException("Destination.Uri cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.Method))
            {
                throw new InvalidOperationException("Destination.Method cannot be null");
            }

            if (!Directory.Exists(this.Source.FilePath))
            {
                throw new InvalidOperationException("Source directory should exist");
            }

            foreach (string file in Directory.GetFiles(this.Source.FilePath))
            {
                var webRequest = CreateRequest(Path.Combine(this.Destination.Uri, Path.GetFileName(file)));
                UploadFile(webRequest, file);
            }
            return new EtlStepResult(EtlStatus.Succeeded, null);
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);
            return result;
        }
 private static object GetSpecialValue(ParameterInfo parameterInfo, EtlContext context, IEtlLogger logger)
 {
     if (parameterInfo.ParameterType == typeof(IEtlLogger))
     {
         return(logger);
     }
     else if (parameterInfo.ParameterType == typeof(EtlContext))
     {
         return(context);
     }
     else
     {
         return(null);
     }
 }
Example #9
0
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            return(new EtlStepResult(EtlStatus.Failed, this.Message));
        }
Example #10
0
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException("Destination cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException("Source.FilePath cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.FilePath))
            {
                throw new InvalidOperationException("Destination.FilePath cannot be empty");
            }

            var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);

            if (!string.IsNullOrEmpty(destinationFolderPath))
            {
                if (!Directory.Exists(destinationFolderPath))
                {
                    Directory.CreateDirectory(destinationFolderPath);
                }
            }

            File.Copy(this.Source.FilePath, this.Destination.FilePath);

            return(new EtlStepResult(EtlStatus.Succeeded, null));
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException("Destination cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException("Source.FilePath cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.FilePath))
            {
                throw new InvalidOperationException("Destination.FilePath cannot be empty");
            }

            var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);

            if (!string.IsNullOrEmpty(destinationFolderPath))
            {
                if (!Directory.Exists(destinationFolderPath))
                {
                    Directory.CreateDirectory(destinationFolderPath);
                }
            }

            File.Copy(this.Source.FilePath, this.Destination.FilePath);

            return new EtlStepResult(EtlStatus.Succeeded, null);
        }
Example #12
0
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.DelayMilliseconds > 0)
            {
                Thread.Sleep(this.DelayMilliseconds);
            }

            return(new EtlStepResult(EtlStatus.Succeeded, null));
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.DelayMilliseconds > 0)
            {
                Thread.Sleep(this.DelayMilliseconds);
            }

            return new EtlStepResult(EtlStatus.Succeeded, null);
        }
Example #14
0
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.ConnectionString))
            {
                throw new InvalidOperationException("Source.ConnectionString cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.ProviderName))
            {
                throw new InvalidOperationException("Source.ProviderName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.Text))
            {
                throw new InvalidOperationException("Source.Text cannot be empty");
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var rowCounter = (this.Counters != null && this.Counters.RowCount != null && !string.IsNullOrEmpty(this.Counters.RowCount.CounterName))
                ? new EtlCounter
            {
                EtlPackageId = context.EtlPackageId,
                EtlSessionId = context.EtlSessionId,
                EntityName   = this.Counters.RowCount.EntityName,
                CounterName  = this.Counters.RowCount.CounterName,
            }
                : null;

            var hasOutputVariables = this.OutputVariables != null && this.OutputVariables.FirstRow.Count > 0;
            var hasRowCounter      = rowCounter != null;

            using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
            {
                if (hasOutputVariables || hasRowCounter)
                {
                    using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
                    {
                        if (dbReader.Read())
                        {
                            if (hasRowCounter)
                            {
                                rowCounter.CounterValue++;
                            }

                            if (hasOutputVariables)
                            {
                                foreach (var firstRowResult in this.OutputVariables.FirstRow)
                                {
                                    var firstRowResultValue = EtlValueTranslation.Evaluate(firstRowResult.SourceFieldName, firstRowResult.SourceFieldTranslation, dbReader, firstRowResult.DefaultValue);
                                    result.VariableAssignments.Add(new EtlVariableAssignment(firstRowResult.VariableName, EtlValueConverter.ToString(firstRowResultValue)));
                                }
                            }
                        }

                        if (hasRowCounter)
                        {
                            while (dbReader.Read())
                            {
                                rowCounter.CounterValue++;
                            }

                            rowCounter.DateTime    = DateTime.Now;
                            rowCounter.UtcDateTime = rowCounter.DateTime.ToUniversalTime();
                            logger.LogEtlCounter(rowCounter);
                        }
                    }
                }
                else
                {
                    dbAccessor.ExecuteNonQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds);
                }
            }

            return(result);
        }
Example #15
0
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException("Source.FilePath cannot be empty");
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException("Destination cannot be null");
            }

            if (string.IsNullOrEmpty(this.Destination.Uri))
            {
                throw new InvalidOperationException("Destination.Uri cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.Method))
            {
                throw new InvalidOperationException("Destination.Method cannot be null");
            }

            if (!File.Exists(this.Source.FilePath))
            {
                //throw exception
            }

            var webRequest = WebRequest.Create(this.Destination.Uri);

            webRequest.Method = this.Destination.Method;

            if (!string.IsNullOrEmpty(this.Destination.ContentType))
            {
                webRequest.ContentType = this.Destination.ContentType;
            }

            foreach (var header in this.Destination.Headers)
            {
                webRequest.Headers[header.Name] = header.Value;
            }

            if (this.TimeoutMilliseconds != null)
            {
                webRequest.Timeout = this.TimeoutMilliseconds.Value == 0 ? System.Threading.Timeout.Infinite : this.TimeoutMilliseconds.Value;
            }

            if (this.Destination.Credential != null && !string.IsNullOrEmpty(this.Destination.Credential.UserName))
            {
                var credential       = new NetworkCredential(this.Destination.Credential.UserName, this.Destination.Credential.Password);
                var credentialsCache = new CredentialCache();
                credentialsCache.Add(new Uri(this.Destination.Uri), AUTHENTICATION_TYPE, credential);
                webRequest.Credentials = credentialsCache;
            }

            if (this.Destination.AllowInvalidCertificates)
            {
                ServicePointManager.ServerCertificateValidationCallback +=
                    delegate
                    (
                        object sender,
                        X509Certificate certificate,
                        X509Chain chain,
                        SslPolicyErrors policyErrors
                    )
                {
                    return(true);
                };
            }

            byte[] fileBytes;

            using (var sourceStream = new StreamReader(this.Source.FilePath))
            {
                var encoding = Encoding.GetEncoding(this.Source.CodePage);
                fileBytes = encoding.GetBytes(sourceStream.ReadToEnd());
                sourceStream.Close();
            }

            webRequest.ContentLength = fileBytes.Length;

            var requestStream = webRequest.GetRequestStream();

            requestStream.Write(fileBytes, 0, fileBytes.Length);
            requestStream.Close();

            return(new EtlStepResult(EtlStatus.Succeeded, null));
        }
Example #16
0
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException("Destination cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.ConnectionString))
            {
                throw new InvalidOperationException("Source.ConnectionString cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.ProviderName))
            {
                throw new InvalidOperationException("Source.ProviderName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.Text))
            {
                throw new InvalidOperationException("Source.Text cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.ConnectionString))
            {
                throw new InvalidOperationException("Destination.ConnectionString cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.ProviderName))
            {
                throw new InvalidOperationException("Destination.ProviderName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.TableName))
            {
                throw new InvalidOperationException("Destination.TableName cannot be empty");
            }

            if (this.Mappings.Count == 0)
            {
                throw new InvalidOperationException("Mappings not specified");
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount   = 0L;
            var insertedRowCount = 0L;
            var errorRowCount    = 0L;

            using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
            {
                using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
                {
                    using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
                    {
                        var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
                        dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
                        {
                            var errorDateTime = DateTime.Now;
                            result.Status  = EtlStatus.FinishedWithLosses;
                            result.Message = e.Message;

                            errorRowCount++;

                            logger.LogEtlMessage
                            (
                                new EtlMessage
                            {
                                EtlPackageId   = context.EtlPackageId,
                                EtlSessionId   = context.EtlSessionId,
                                EtlStepName    = this.Name,
                                LogDateTime    = errorDateTime,
                                LogUtcDateTime = errorDateTime.ToUniversalTime(),
                                MessageType    = EtlMessageType.Error,
                                Text           = e.Message,
                                Flags          = e.RecordIndex,
                                StackTrace     = e.Exception != null ? e.Exception.StackTrace : null,
                            }
                            );

                            if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
                            {
                                result.Status  = EtlStatus.FinishedWithLosses;
                                e.TrySkipError = true;
                            }
                            else
                            {
                                result.Status = EtlStatus.Failed;
                            }
                        };

                        insertedRowCount = dbWriter.Write(mapReader, this.TimeoutMilliseconds, this.BatchSize);
                    }
                }
            }

            sourceRowCount = insertedRowCount + errorRowCount;

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Found",
            //    Flags = sourceRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Errors",
            //    Flags = errorRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Inserted",
            //    Flags = insertedRowCount,
            //});

            return(result);
        }
 private static object GetSpecialValue(ParameterInfo parameterInfo, EtlContext context, IEtlLogger logger)
 {
     if (parameterInfo.ParameterType == typeof(IEtlLogger))
     {
         return logger;
     }
     else if (parameterInfo.ParameterType == typeof(EtlContext))
     {
         return context;
     }
     else
     {
         return null;
     }
 }
Example #18
0
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
            }

            if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
            }

            if (this.Source.FieldDelimiter.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
            }

            if (this.Source.Quote != null && this.Source.Quote.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
            }

            if (this.Source.Escape != null && this.Source.Escape.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount = 0L;
            var validRowCount  = 0L;
            var errorRowCount  = 0L;

            var csvSyntax = new CsvSyntaxInfo
            {
                HasHeaders     = this.Source.HasHeaders,
                FieldDelimiter = this.Source.FieldDelimiter[0],
                Quote          = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
                Escape         = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
                LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
                LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
            };

            using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
            {
                using (var csvReader = new CsvReader(fileReader, csvSyntax))
                {
                    var validator = new CsvValidator(this, csvReader, context, logger);
                    result.Status  = validator.Validate();
                    sourceRowCount = validator.ReadRowCount;
                    validRowCount  = validator.ValidRowCount;
                    errorRowCount  = validator.ErrorRowCount;
                }
            }

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Found",
            //    Flags = sourceRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Validated",
            //    Flags = validRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Errors",
            //    Flags = errorRowCount,
            //});

            return(result);
        }
Example #19
0
        public static ExecutionResult Execute(CommandContext commandContext, CompiledModule module, string[] commands)
        {
            var result = ExecutionResult.Success;

            var etlContext = new EtlContext();

            try
            {
                var listeners = commandContext.HostConfiguration.GetSessionListeners(null);
                if (listeners?.Count > 0)
                {
                    etlContext.Listeners.AddRange(listeners);
                }
            }
            catch (Exception ex)
            {
                var formattedMessage = ex.FormatExceptionWithDetails();
                etlContext.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
                etlContext.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
            }

            var instance  = Environment.MachineName;
            var arguments = GetArguments(module, instance);

            var environmentSettings = new EnvironmentSettings();

            module.Startup.Configure(environmentSettings);
            var taskCreators = new Dictionary <string, Func <IEtlSessionArguments, IEtlTask> >(module.Startup.Commands, StringComparer.InvariantCultureIgnoreCase);

            var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
            var session   = new EtlSession(sessionId, etlContext, arguments);

            etlContext.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;

            var logger    = CreateLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
            var opsLogger = CreateOpsLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
            var ioLogger  = CreateIoLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);

            var serilogAdapter = new EtlSessionSerilogAdapter(logger, opsLogger, ioLogger, commandContext.DevLogFolder, commandContext.OpsLogFolder);

            etlContext.Listeners.Add(serilogAdapter);

            serilogAdapter.Log(LogSeverity.Information, false, null, null, "session {SessionId} started", sessionId);

            if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
            {
                etlContext.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
            }

            var sessionStartedOn      = Stopwatch.StartNew();
            var sessionExceptionCount = 0;

            var taskResults = new List <KeyValuePair <IEtlTask, TaskResult> >();

            try
            {
                foreach (var command in commands)
                {
                    IEtlTask task = null;
                    if (taskCreators.TryGetValue(command, out var taskCreator))
                    {
                        task = taskCreator.Invoke(arguments);
                    }
                    else
                    {
                        var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, command, StringComparison.InvariantCultureIgnoreCase));
                        if (taskType != null)
                        {
                            task = (IEtlTask)Activator.CreateInstance(taskType);
                        }
                    }

                    if (task == null)
                    {
                        serilogAdapter.Log(LogSeverity.Error, false, null, null, "unknown task/flow type: " + command);
                        break;
                    }

                    try
                    {
                        try
                        {
                            var taskResult = session.ExecuteTask(null, task);
                            taskResults.Add(new KeyValuePair <IEtlTask, TaskResult>(task, taskResult));

                            sessionExceptionCount += taskResult.ExceptionCount;

                            if (sessionExceptionCount > 0)
                            {
                                etlContext.Log(LogSeverity.Error, task, "failed, terminating execution");
                                result = ExecutionResult.ExecutionFailed;
                                etlContext.Close();
                                break; // stop processing tasks
                            }
                        }
                        catch (Exception ex)
                        {
                            etlContext.Log(LogSeverity.Error, task, "failed, terminating execution, reason: ", task.Statistics.RunTime, ex.Message);
                            result = ExecutionResult.ExecutionFailed;
                            etlContext.Close();
                            break; // stop processing tasks
                        }
                    }
                    catch (TransactionAbortedException)
                    {
                    }

                    LogTaskCounters(serilogAdapter, task);
                }

                session.Stop();

                if (taskResults.Count > 0)
                {
                    serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
                    serilogAdapter.Log(LogSeverity.Information, false, null, null, "SUMMARY");
                    serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");

                    var longestTaskName = taskResults.Max(x => x.Key.Name.Length);
                    foreach (var kvp in taskResults)
                    {
                        LogTaskSummary(serilogAdapter, kvp.Key, kvp.Value, longestTaskName);
                    }
                }

                etlContext.Close();
            }
            catch (TransactionAbortedException)
            {
            }

            return(result);
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
            }

            if (string.IsNullOrEmpty(this.Source.ConnectionString))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
            }

            if (string.IsNullOrEmpty(this.Source.ProviderName))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.ProviderName"));
            }

            if (string.IsNullOrEmpty(this.Source.Text))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.Text"));
            }

            if (string.IsNullOrEmpty(this.Destination.FieldDelimiter))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FieldDelimiter"));
            }

            if (string.IsNullOrEmpty(this.Destination.FilePath))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FilePath"));
            }

            if (this.Destination.FieldDelimiter.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.FieldDelimiter", 1));
            }

            if (this.Destination.Quote != null && this.Destination.Quote.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Quote", 1));
            }

            if (this.Destination.Escape != null && this.Destination.Escape.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Escape", 1));
            }

            if (this.Destination != null)
            {
                var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);

                if (!string.IsNullOrEmpty(destinationFolderPath))
                {
                    if (!Directory.Exists(destinationFolderPath))
                    {
                        Directory.CreateDirectory(destinationFolderPath);
                    }
                }
            }

            //if (this.Mappings.Count == 0)
            //{
            //    throw new InvalidOperationException("Mappings not specified");
            //}

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount = 0L;
            var writtenRowCount = 0L;

            var encoding = Encoding.GetEncoding(this.Destination.CodePage);
            var csvSyntax = new CsvSyntaxInfo
            {
                HasHeaders = this.Destination.HasHeaders,
                FieldDelimiter = this.Destination.FieldDelimiter[0],
                Quote = this.Destination.Quote != null ? this.Destination.Quote[0] : DefaultQuote,
                Escape = this.Destination.Escape != null ? this.Destination.Escape[0] : DefaultEscape,
                LineDelimiter1 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[0] : DefaultLineDelimiter[0],
                LineDelimiter2 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[1] : DefaultLineDelimiter[1],
            };

            using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
            {
                using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
                {
                    using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
                    {
                        using (var fileWriter = new StreamWriter(this.Destination.FilePath, false, encoding))
                        {
                            var csvWriter = new CsvWriter(fileWriter, csvSyntax);

                            writtenRowCount = csvWriter.Write(mapReader);
                            sourceRowCount = writtenRowCount;
                        }
                    }
                }
            }

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Found",
            //    Flags = sourceRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Exported",
            //    Flags = writtenRowCount,
            //});

            return result;
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.Uri))
            {
                throw new InvalidOperationException("Source.Uri cannot be empty");
            }

            if (this.Destination != null && string.IsNullOrEmpty(this.Destination.FilePath))
            {
                throw new InvalidOperationException("Destination.FilePath cannot be empty");
            }

            if (this.Destination != null)
            {
                var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);

                if (!string.IsNullOrEmpty(destinationFolderPath))
                {
                    if (!Directory.Exists(destinationFolderPath))
                    {
                        Directory.CreateDirectory(destinationFolderPath);
                    }
                }
            }

            //получение списка файлов
            var listDirWebRequest = GetRequest(WebRequestMethods.Ftp.ListDirectory, this.Source.Uri);

            if (this.Source.AllowInvalidCertificates)
            {
                ServicePointManager.ServerCertificateValidationCallback +=
                delegate
                (
                    object sender,
                    X509Certificate certificate,
                    X509Chain chain,
                    SslPolicyErrors policyErrors
                )
                {
                    return true;
                };
            }

            if (!string.IsNullOrEmpty(this.Source.Request))
            {
                var bytes = Encoding.UTF8.GetBytes(this.Source.Request);
                listDirWebRequest.ContentLength = bytes.Length;

                var request = listDirWebRequest.GetRequestStream();
                request.Write(bytes, 0, bytes.Length);
                request.Close();
            }

            var files = GetFileList(listDirWebRequest);

            //получение файлов
            foreach (var file in files)
            {
                var fileRequest = GetRequest(WebRequestMethods.Ftp.DownloadFile, this.Source.Uri+"/"+file);
                using (var response = fileRequest.GetResponse())
                {
                    if (this.Destination != null)
                    {
                        using (var responseStream = response.GetResponseStream())
                        {
                            using (var fileStream = File.Create(this.Destination.FilePath+"\\"+file))
                            {
                                var buffer = new byte[1024];
                                var bytesRead = 0;
                                //var bytesProcessed = 0;

                                do
                                {
                                    bytesRead = responseStream.Read(buffer, 0, buffer.Length);
                                    fileStream.Write(buffer, 0, bytesRead);
                                    //bytesProcessed += bytesRead;
                                } while (bytesRead > 0);
                            }
                        }
                    }
                }
            }

            return new EtlStepResult(EtlStatus.Succeeded, null);
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.AssemblyName))
            {
                throw new InvalidOperationException("Source.Assembly cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.TypeName))
            {
                throw new InvalidOperationException("Source.TypeName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.MethodName))
            {
                throw new InvalidOperationException("Source.MethodName cannot be empty");
            }

            var result     = new EtlStepResult(EtlStatus.Succeeded, null);
            var sourcePath = string.Format("{0}, {1}, {2}", this.Source.AssemblyName, this.Source.TypeName, this.Source.MethodName);

            var asm = Assembly.Load(this.Source.AssemblyName);

            var type = asm.GetType(this.Source.TypeName);

            if (type == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.InvokeMethodCannotFindType, this.Source.AssemblyName, this.Source.TypeName));
            }

            var method = type.GetMethod(this.Source.MethodName);

            if (method == null)
            {
                throw new InvalidOperationException("Method '" + Source.MethodName + "' does not exist within assembly " + Source.AssemblyName);
            }

            if (!method.IsStatic)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.InvokeMethodCannotInvokeNonStaticMethod, this.Source.AssemblyName, this.Source.TypeName, this.Source.MethodName));
            }

            var values = GetParameterValues(method, this.Source.Parameters, context, logger);
            var res    = method.Invoke(null, values);

            if (res is EtlStepResult)
            {
                result = (EtlStepResult)res;
            }

            return(result);
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.Uri))
            {
                throw new InvalidOperationException("Source.Uri cannot be empty");
            }

            if (this.Destination != null && string.IsNullOrEmpty(this.Destination.FilePath))
            {
                throw new InvalidOperationException("Destination.FilePath cannot be empty");
            }

            if (this.Destination != null)
            {
                var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);

                if (!string.IsNullOrEmpty(destinationFolderPath))
                {
                    if (!Directory.Exists(destinationFolderPath))
                    {
                        Directory.CreateDirectory(destinationFolderPath);
                    }
                }
            }


            //получение списка файлов
            var listDirWebRequest = GetRequest(WebRequestMethods.Ftp.ListDirectory, this.Source.Uri);

            if (this.Source.AllowInvalidCertificates)
            {
                ServicePointManager.ServerCertificateValidationCallback +=
                    delegate
                    (
                        object sender,
                        X509Certificate certificate,
                        X509Chain chain,
                        SslPolicyErrors policyErrors
                    )
                {
                    return(true);
                };
            }

            if (!string.IsNullOrEmpty(this.Source.Request))
            {
                var bytes = Encoding.UTF8.GetBytes(this.Source.Request);
                listDirWebRequest.ContentLength = bytes.Length;

                var request = listDirWebRequest.GetRequestStream();
                request.Write(bytes, 0, bytes.Length);
                request.Close();
            }

            var files = GetFileList(listDirWebRequest);


            //получение файлов
            foreach (var file in files)
            {
                var fileRequest = GetRequest(WebRequestMethods.Ftp.DownloadFile, this.Source.Uri + "/" + file);
                using (var response = fileRequest.GetResponse())
                {
                    if (this.Destination != null)
                    {
                        using (var responseStream = response.GetResponseStream())
                        {
                            using (var fileStream = File.Create(this.Destination.FilePath + "\\" + file))
                            {
                                var buffer    = new byte[1024];
                                var bytesRead = 0;
                                //var bytesProcessed = 0;

                                do
                                {
                                    bytesRead = responseStream.Read(buffer, 0, buffer.Length);
                                    fileStream.Write(buffer, 0, bytesRead);
                                    //bytesProcessed += bytesRead;
                                } while (bytesRead > 0);
                            }
                        }
                    }
                }
            }

            return(new EtlStepResult(EtlStatus.Succeeded, null));
        }
        private static object[] GetParameterValues(MethodInfo method, IEnumerable<EtlMethodParameter> parameters, EtlContext context, IEtlLogger logger)
        {
            var parametersInfo = method.GetParameters();
            var values = new object[parametersInfo.Length];

            for (var i = 0; i < parametersInfo.Length; i++)
            {
                var parameterInfo = parametersInfo[i];
                var parameter = FindParameter(parameters, parameterInfo.Name);
                if (parameter == null)
                {
                    values[i] = GetSpecialValue(parameterInfo, context, logger);
                }
                else
                {
                    values[i] = EtlValueConverter.ParseType(parameter.Value, Type.GetTypeCode(parameterInfo.ParameterType));
                }
            }

            return values;
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
            }

            if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
            }

            if (this.Source.FieldDelimiter.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
            }

            if (this.Source.Quote != null && this.Source.Quote.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
            }

            if (this.Source.Escape != null && this.Source.Escape.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount = 0L;
            var validRowCount = 0L;
            var errorRowCount = 0L;

            var csvSyntax = new CsvSyntaxInfo
            {
                HasHeaders = this.Source.HasHeaders,
                FieldDelimiter = this.Source.FieldDelimiter[0],
                Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
                Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
                LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
                LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
            };

            using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
            {
                using (var csvReader = new CsvReader(fileReader, csvSyntax))
                {
                    var validator = new CsvValidator(this, csvReader, context, logger);
                    result.Status = validator.Validate();
                    sourceRowCount = validator.ReadRowCount;
                    validRowCount = validator.ValidRowCount;
                    errorRowCount = validator.ErrorRowCount;
                }
            }

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Found",
            //    Flags = sourceRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Validated",
            //    Flags = validRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Errors",
            //    Flags = errorRowCount,
            //});

            return result;
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.AssemblyName))
            {
                throw new InvalidOperationException("Source.Assembly cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.TypeName))
            {
                throw new InvalidOperationException("Source.TypeName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.MethodName))
            {
                throw new InvalidOperationException("Source.MethodName cannot be empty");
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);
            var sourcePath = string.Format("{0}, {1}, {2}", this.Source.AssemblyName, this.Source.TypeName, this.Source.MethodName);

            var asm = Assembly.Load(this.Source.AssemblyName);

            var type = asm.GetType(this.Source.TypeName);
            if (type == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.InvokeMethodCannotFindType, this.Source.AssemblyName, this.Source.TypeName));
            }

            var method = type.GetMethod(this.Source.MethodName);
            if (method == null)
            {
                throw new InvalidOperationException("Method '" + Source.MethodName + "' does not exist within assembly " + Source.AssemblyName);
            }

            if (!method.IsStatic)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.InvokeMethodCannotInvokeNonStaticMethod, this.Source.AssemblyName, this.Source.TypeName, this.Source.MethodName));
            }

            var values = GetParameterValues(method, this.Source.Parameters, context, logger);
            var res = method.Invoke(null, values);

            if (res is EtlStepResult)
            {
                result = (EtlStepResult) res;
            }

            return result;
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
            }

            if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
            }

            if (string.IsNullOrEmpty(this.Destination.ConnectionString))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
            }

            if (string.IsNullOrEmpty(this.Destination.ProviderName))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ProviderName"));
            }

            if (string.IsNullOrEmpty(this.Destination.TableName))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.TableName"));
            }

            if (this.Source.FieldDelimiter.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
            }

            if (this.Source.Quote != null && this.Source.Quote.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
            }

            if (this.Source.Escape != null && this.Source.Escape.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount = 0L;
            var insertedRowCount = 0L;
            var errorRowCount = 0L;

            var csvSyntax = new CsvSyntaxInfo
            {
                HasHeaders = this.Source.HasHeaders,
                FieldDelimiter = this.Source.FieldDelimiter[0],
                Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
                Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
                LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
                LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
            };

            using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
            {
                using (var csvReader = new CsvReader(fileReader, csvSyntax))
                {
                    var wrappedReader = new EtlMappedDataReader(csvReader, this.Mappings);

                    var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
                    dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
                    {
                        var errorDateTime = DateTime.Now;
                        result.Status = EtlStatus.FinishedWithLosses;
                        result.Message = e.Message;

                        //todo: fix error counter in CSV import. Now it counts errors, not error records
                        errorRowCount++;

                        logger.LogEtlMessage
                        (
                            new EtlMessage
                            {
                                EtlPackageId = context.EtlPackageId,
                                EtlSessionId = context.EtlSessionId,
                                EtlStepName = this.Name,
                                LogDateTime = errorDateTime,
                                LogUtcDateTime = errorDateTime.ToUniversalTime(),
                                MessageType = EtlMessageType.Error,
                                Text = e.Message,
                                Flags = e.RecordIndex,
                                StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
                            }
                        );

                        if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
                        {
                            result.Status = EtlStatus.FinishedWithLosses;
                            e.TrySkipError = true;
                        }
                        else
                        {
                            result.Status = EtlStatus.Failed;
                        }
                    };

                    insertedRowCount = dbWriter.Write(wrappedReader, this.TimeoutMilliseconds, this.BatchSize);
                    sourceRowCount = csvReader.CurrentRecordIndex + 1;
                }
            }

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Errors",
            //    Flags = errorRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Inserted",
            //    Flags = insertedRowCount,
            //});

            return result;
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException("Source.FilePath cannot be empty");
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException("Destination cannot be null");
            }

            if (string.IsNullOrEmpty(this.Destination.Uri))
            {
                throw new InvalidOperationException("Destination.Uri cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.Method))
            {
                throw new InvalidOperationException("Destination.Method cannot be null");
            }

            if (!File.Exists(this.Source.FilePath))
            {
                //throw exception
            }

            var webRequest = WebRequest.Create(this.Destination.Uri);
            webRequest.Method = this.Destination.Method;

            if (!string.IsNullOrEmpty(this.Destination.ContentType))
            {
                webRequest.ContentType = this.Destination.ContentType;
            }

            foreach (var header in this.Destination.Headers)
            {
                webRequest.Headers[header.Name] = header.Value;
            }

            if (this.TimeoutMilliseconds != null)
            {
                webRequest.Timeout = this.TimeoutMilliseconds.Value == 0 ? System.Threading.Timeout.Infinite : this.TimeoutMilliseconds.Value;
            }

            if (this.Destination.Credential != null && !string.IsNullOrEmpty(this.Destination.Credential.UserName))
            {
                var credential = new NetworkCredential(this.Destination.Credential.UserName, this.Destination.Credential.Password);
                var credentialsCache = new CredentialCache();
                credentialsCache.Add(new Uri(this.Destination.Uri), AUTHENTICATION_TYPE, credential);
                webRequest.Credentials = credentialsCache;
            }

            if (this.Destination.AllowInvalidCertificates)
            {
                ServicePointManager.ServerCertificateValidationCallback +=
                delegate
                (
                    object sender,
                    X509Certificate certificate,
                    X509Chain chain,
                    SslPolicyErrors policyErrors
                )
                {
                    return true;
                };
            }

            byte[] fileBytes;

            using (var sourceStream = new StreamReader(this.Source.FilePath))
            {
                var encoding = Encoding.GetEncoding(this.Source.CodePage);
                fileBytes = encoding.GetBytes(sourceStream.ReadToEnd());
                sourceStream.Close();
            }

            webRequest.ContentLength = fileBytes.Length;

            var requestStream = webRequest.GetRequestStream();
            requestStream.Write(fileBytes, 0, fileBytes.Length);
            requestStream.Close();

            return new EtlStepResult(EtlStatus.Succeeded, null);
        }
        private static object[] GetParameterValues(MethodInfo method, IEnumerable <EtlMethodParameter> parameters, EtlContext context, IEtlLogger logger)
        {
            var parametersInfo = method.GetParameters();
            var values         = new object[parametersInfo.Length];

            for (var i = 0; i < parametersInfo.Length; i++)
            {
                var parameterInfo = parametersInfo[i];
                var parameter     = FindParameter(parameters, parameterInfo.Name);
                if (parameter == null)
                {
                    values[i] = GetSpecialValue(parameterInfo, context, logger);
                }
                else
                {
                    values[i] = EtlValueConverter.ParseType(parameter.Value, Type.GetTypeCode(parameterInfo.ParameterType));
                }
            }

            return(values);
        }
Example #30
0
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.Uri))
            {
                throw new InvalidOperationException("Source.Uri cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.Method))
            {
                throw new InvalidOperationException("Source.Method cannot be null");
            }

            if (this.Destination != null && string.IsNullOrEmpty(this.Destination.FilePath))
            {
                throw new InvalidOperationException("Destination.FilePath cannot be empty");
            }

            if (this.Destination != null)
            {
                var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);

                if (!string.IsNullOrEmpty(destinationFolderPath))
                {
                    if (!Directory.Exists(destinationFolderPath))
                    {
                        Directory.CreateDirectory(destinationFolderPath);
                    }
                }
            }

            var webRequest = WebRequest.Create(this.Source.Uri);

            webRequest.Method = this.Source.Method;

            if (!string.IsNullOrEmpty(this.Source.ContentType))
            {
                webRequest.ContentType = this.Source.ContentType;
            }

            foreach (var header in this.Source.Headers)
            {
                webRequest.Headers[header.Name] = header.Value;
            }

            if (this.TimeoutMilliseconds != null)
            {
                webRequest.Timeout = this.TimeoutMilliseconds.Value == 0 ? System.Threading.Timeout.Infinite : this.TimeoutMilliseconds.Value;
            }

            if (this.Source.Credential != null && !string.IsNullOrEmpty(this.Source.Credential.UserName))
            {
                var credential       = new NetworkCredential(this.Source.Credential.UserName, this.Source.Credential.Password);
                var credentialsCache = new CredentialCache();
                credentialsCache.Add(new Uri(this.Source.Uri), AUTHENTICATION_TYPE, credential);
                webRequest.Credentials = credentialsCache;
            }

            if (this.Source.AllowInvalidCertificates)
            {
                ServicePointManager.ServerCertificateValidationCallback +=
                    delegate
                    (
                        object sender,
                        X509Certificate certificate,
                        X509Chain chain,
                        SslPolicyErrors policyErrors
                    )
                {
                    return(true);
                };
            }

            if (!string.IsNullOrEmpty(this.Source.Request))
            {
                var bytes = Encoding.UTF8.GetBytes(this.Source.Request);
                webRequest.ContentLength = bytes.Length;

                var request = webRequest.GetRequestStream();
                request.Write(bytes, 0, bytes.Length);
                request.Close();
            }

            using (var response = webRequest.GetResponse())
            {
                if (this.Destination != null)
                {
                    using (var responseStream = response.GetResponseStream())
                    {
                        using (var fileStream = File.Create(this.Destination.FilePath))
                        {
                            var buffer    = new byte[1024];
                            var bytesRead = 0;
                            //var bytesProcessed = 0;

                            do
                            {
                                bytesRead = responseStream.Read(buffer, 0, buffer.Length);
                                fileStream.Write(buffer, 0, bytesRead);
                                //bytesProcessed += bytesRead;
                            } while (bytesRead > 0);
                        }
                    }
                }
            }

            return(new EtlStepResult(EtlStatus.Succeeded, null));
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
            }

            if (string.IsNullOrEmpty(this.Source.ConnectionString))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
            }

            if (string.IsNullOrEmpty(this.Source.ProviderName))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.ProviderName"));
            }

            if (string.IsNullOrEmpty(this.Source.Text))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.Text"));
            }

            if (string.IsNullOrEmpty(this.Destination.FieldDelimiter))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FieldDelimiter"));
            }

            if (string.IsNullOrEmpty(this.Destination.FilePath))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FilePath"));
            }

            if (this.Destination.FieldDelimiter.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.FieldDelimiter", 1));
            }

            if (this.Destination.Quote != null && this.Destination.Quote.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Quote", 1));
            }

            if (this.Destination.Escape != null && this.Destination.Escape.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Escape", 1));
            }

            if (this.Destination != null)
            {
                var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);

                if (!string.IsNullOrEmpty(destinationFolderPath))
                {
                    if (!Directory.Exists(destinationFolderPath))
                    {
                        Directory.CreateDirectory(destinationFolderPath);
                    }
                }
            }

            //if (this.Mappings.Count == 0)
            //{
            //    throw new InvalidOperationException("Mappings not specified");
            //}

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount  = 0L;
            var writtenRowCount = 0L;

            var encoding  = Encoding.GetEncoding(this.Destination.CodePage);
            var csvSyntax = new CsvSyntaxInfo
            {
                HasHeaders     = this.Destination.HasHeaders,
                FieldDelimiter = this.Destination.FieldDelimiter[0],
                Quote          = this.Destination.Quote != null ? this.Destination.Quote[0] : DefaultQuote,
                Escape         = this.Destination.Escape != null ? this.Destination.Escape[0] : DefaultEscape,
                LineDelimiter1 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[0] : DefaultLineDelimiter[0],
                LineDelimiter2 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[1] : DefaultLineDelimiter[1],
            };

            using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
            {
                using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
                {
                    using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
                    {
                        using (var fileWriter = new StreamWriter(this.Destination.FilePath, false, encoding))
                        {
                            var csvWriter = new CsvWriter(fileWriter, csvSyntax);

                            writtenRowCount = csvWriter.Write(mapReader);
                            sourceRowCount  = writtenRowCount;
                        }
                    }
                }
            }

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Found",
            //    Flags = sourceRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Exported",
            //    Flags = writtenRowCount,
            //});

            return(result);
        }
Example #32
0
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
            }

            if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
            }

            if (string.IsNullOrEmpty(this.Destination.ConnectionString))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
            }

            if (string.IsNullOrEmpty(this.Destination.ProviderName))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ProviderName"));
            }

            if (string.IsNullOrEmpty(this.Destination.TableName))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.TableName"));
            }

            if (this.Source.FieldDelimiter.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
            }

            if (this.Source.Quote != null && this.Source.Quote.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
            }

            if (this.Source.Escape != null && this.Source.Escape.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount   = 0L;
            var insertedRowCount = 0L;
            var errorRowCount    = 0L;

            var csvSyntax = new CsvSyntaxInfo
            {
                HasHeaders     = this.Source.HasHeaders,
                FieldDelimiter = this.Source.FieldDelimiter[0],
                Quote          = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
                Escape         = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
                LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
                LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
            };

            using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
            {
                using (var csvReader = new CsvReader(fileReader, csvSyntax))
                {
                    var wrappedReader = new EtlMappedDataReader(csvReader, this.Mappings);

                    var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
                    dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
                    {
                        var errorDateTime = DateTime.Now;
                        result.Status  = EtlStatus.FinishedWithLosses;
                        result.Message = e.Message;

                        //todo: fix error counter in CSV import. Now it counts errors, not error records
                        errorRowCount++;

                        logger.LogEtlMessage
                        (
                            new EtlMessage
                        {
                            EtlPackageId   = context.EtlPackageId,
                            EtlSessionId   = context.EtlSessionId,
                            EtlStepName    = this.Name,
                            LogDateTime    = errorDateTime,
                            LogUtcDateTime = errorDateTime.ToUniversalTime(),
                            MessageType    = EtlMessageType.Error,
                            Text           = e.Message,
                            Flags          = e.RecordIndex,
                            StackTrace     = e.Exception != null ? e.Exception.StackTrace : null,
                        }
                        );

                        if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
                        {
                            result.Status  = EtlStatus.FinishedWithLosses;
                            e.TrySkipError = true;
                        }
                        else
                        {
                            result.Status = EtlStatus.Failed;
                        }
                    };

                    insertedRowCount = dbWriter.Write(wrappedReader, this.TimeoutMilliseconds, this.BatchSize);
                    sourceRowCount   = csvReader.CurrentRecordIndex + 1;
                }
            }

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Errors",
            //    Flags = errorRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Inserted",
            //    Flags = insertedRowCount,
            //});

            return(result);
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException("Destination cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.ConnectionString))
            {
                throw new InvalidOperationException("Source.ConnectionString cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.ProviderName))
            {
                throw new InvalidOperationException("Source.ProviderName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.Text))
            {
                throw new InvalidOperationException("Source.Text cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.ConnectionString))
            {
                throw new InvalidOperationException("Destination.ConnectionString cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.ProviderName))
            {
                throw new InvalidOperationException("Destination.ProviderName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Destination.TableName))
            {
                throw new InvalidOperationException("Destination.TableName cannot be empty");
            }

            if (this.Mappings.Count == 0)
            {
                throw new InvalidOperationException("Mappings not specified");
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount = 0L;
            var insertedRowCount = 0L;
            var errorRowCount = 0L;

            using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
            {
                using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
                {
                    using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
                    {
                        var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
                        dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
                        {
                            var errorDateTime = DateTime.Now;
                            result.Status = EtlStatus.FinishedWithLosses;
                            result.Message = e.Message;

                            errorRowCount++;

                            logger.LogEtlMessage
                            (
                                new EtlMessage
                                {
                                    EtlPackageId = context.EtlPackageId,
                                    EtlSessionId = context.EtlSessionId,
                                    EtlStepName = this.Name,
                                    LogDateTime = errorDateTime,
                                    LogUtcDateTime = errorDateTime.ToUniversalTime(),
                                    MessageType = EtlMessageType.Error,
                                    Text = e.Message,
                                    Flags = e.RecordIndex,
                                    StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
                                }
                            );

                            if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
                            {
                                result.Status = EtlStatus.FinishedWithLosses;
                                e.TrySkipError = true;
                            }
                            else
                            {
                                result.Status = EtlStatus.Failed;
                            }
                        };

                        insertedRowCount = dbWriter.Write(mapReader, this.TimeoutMilliseconds, this.BatchSize);
                    }
                }
            }

            sourceRowCount = insertedRowCount + errorRowCount;

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Found",
            //    Flags = sourceRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Errors",
            //    Flags = errorRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Inserted",
            //    Flags = insertedRowCount,
            //});

            return result;
        }
            public CsvValidator(EtlValidateCsvFileStep step, CsvReader reader, EtlContext context, IEtlLogger logger)
            {
                _step = step;
                _reader = reader;
                _context = context;
                _logger = logger;

                _reader.ParseError += OnReaderError;
            }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.Uri))
            {
                throw new InvalidOperationException("Source.Uri cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.Method))
            {
                throw new InvalidOperationException("Source.Method cannot be null");
            }

            if (this.Destination != null && string.IsNullOrEmpty(this.Destination.FilePath))
            {
                throw new InvalidOperationException("Destination.FilePath cannot be empty");
            }

            if (this.Destination != null)
            {
                var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);

                if (!string.IsNullOrEmpty(destinationFolderPath))
                {
                    if (!Directory.Exists(destinationFolderPath))
                    {
                        Directory.CreateDirectory(destinationFolderPath);
                    }
                }
            }

            var webRequest = WebRequest.Create(this.Source.Uri);
            webRequest.Method = this.Source.Method;

            if (!string.IsNullOrEmpty(this.Source.ContentType))
            {
                webRequest.ContentType = this.Source.ContentType;
            }

            foreach (var header in this.Source.Headers)
            {
                webRequest.Headers[header.Name] = header.Value;
            }

            if (this.TimeoutMilliseconds != null)
            {
                webRequest.Timeout = this.TimeoutMilliseconds.Value == 0 ? System.Threading.Timeout.Infinite : this.TimeoutMilliseconds.Value;
            }

            if (this.Source.Credential != null && !string.IsNullOrEmpty(this.Source.Credential.UserName))
            {
                var credential = new NetworkCredential(this.Source.Credential.UserName, this.Source.Credential.Password);
                var credentialsCache = new CredentialCache();
                credentialsCache.Add(new Uri(this.Source.Uri), AUTHENTICATION_TYPE, credential);
                webRequest.Credentials = credentialsCache;
            }

            if (this.Source.AllowInvalidCertificates)
            {
                ServicePointManager.ServerCertificateValidationCallback +=
                delegate
                (
                    object sender,
                    X509Certificate certificate,
                    X509Chain chain,
                    SslPolicyErrors policyErrors
                )
                {
                    return true;
                };
            }

            if (!string.IsNullOrEmpty(this.Source.Request))
            {
                var bytes = Encoding.UTF8.GetBytes(this.Source.Request);
                webRequest.ContentLength = bytes.Length;

                var request = webRequest.GetRequestStream();
                request.Write(bytes, 0, bytes.Length);
                request.Close();
            }

            using (var response = webRequest.GetResponse())
            {
                if (this.Destination != null)
                {
                    using (var responseStream = response.GetResponseStream())
                    {
                        using (var fileStream = File.Create(this.Destination.FilePath))
                        {
                            var buffer = new byte[1024];
                            var bytesRead = 0;
                            //var bytesProcessed = 0;

                            do
                            {
                                bytesRead = responseStream.Read(buffer, 0, buffer.Length);
                                fileStream.Write(buffer, 0, bytesRead);
                                //bytesProcessed += bytesRead;
                            } while (bytesRead > 0);
                        }
                    }
                }
            }

            return new EtlStepResult(EtlStatus.Succeeded, null);
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException("Source cannot be null");
            }

            if (string.IsNullOrEmpty(this.Source.ConnectionString))
            {
                throw new InvalidOperationException("Source.ConnectionString cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.ProviderName))
            {
                throw new InvalidOperationException("Source.ProviderName cannot be empty");
            }

            if (string.IsNullOrEmpty(this.Source.Text))
            {
                throw new InvalidOperationException("Source.Text cannot be empty");
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var rowCounter = (this.Counters != null && this.Counters.RowCount != null && !string.IsNullOrEmpty(this.Counters.RowCount.CounterName))
                ? new EtlCounter
                {
                    EtlPackageId = context.EtlPackageId,
                    EtlSessionId = context.EtlSessionId,
                    EntityName = this.Counters.RowCount.EntityName,
                    CounterName = this.Counters.RowCount.CounterName,
                }
                : null;

            var hasOutputVariables = this.OutputVariables != null && this.OutputVariables.FirstRow.Count > 0;
            var hasRowCounter = rowCounter != null;

            using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
            {
                if (hasOutputVariables || hasRowCounter)
                {
                    using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
                    {
                        if (dbReader.Read())
                        {
                            if (hasRowCounter)
                            {
                                rowCounter.CounterValue++;
                            }

                            if (hasOutputVariables)
                            {
                                foreach (var firstRowResult in this.OutputVariables.FirstRow)
                                {
                                    var firstRowResultValue = EtlValueTranslation.Evaluate(firstRowResult.SourceFieldName, firstRowResult.SourceFieldTranslation, dbReader, firstRowResult.DefaultValue);
                                    result.VariableAssignments.Add(new EtlVariableAssignment(firstRowResult.VariableName, EtlValueConverter.ToString(firstRowResultValue)));
                                }
                            }
                        }

                        if (hasRowCounter)
                        {
                            while (dbReader.Read())
                            {
                                rowCounter.CounterValue++;
                            }

                            rowCounter.DateTime = DateTime.Now;
                            rowCounter.UtcDateTime = rowCounter.DateTime.ToUniversalTime();
                            logger.LogEtlCounter(rowCounter);
                        }
                    }
                }
                else
                {
                    dbAccessor.ExecuteNonQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds);
                }
            }

            return result;
        }
        public static void DoSomething(EtlContext context, IEtlLogger logger, string a, int b, DateTime c, string fileName)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            File.WriteAllText(fileName, string.Format("{0}, {1}, {2:o}", a, b, c));
        }