public CsvWriter(TextWriter writer, CsvSyntaxInfo syntax)
        {
            if (writer == null)
            {
                throw new ArgumentNullException("writer");
            }

            if (syntax == null)
            {
                throw new ArgumentNullException("syntax");
            }

            _writer = writer;

            _fieldDelimiter = syntax.FieldDelimiter.ToString();
            _lineDelimiter = string.Concat(syntax.LineDelimiter1, syntax.LineDelimiter2);
            _quote = syntax.Quote.ToString();
            _escape = syntax.Escape.ToString();

            _escapeAndQuote = string.Concat(_escape, _quote);
            _escapeAndEscape = string.Concat(_escape, _escape);

            _hasHeaders = syntax.HasHeaders;
        }
Beispiel #2
0
        public CsvWriter(TextWriter writer, CsvSyntaxInfo syntax)
        {
            if (writer == null)
            {
                throw new ArgumentNullException("writer");
            }

            if (syntax == null)
            {
                throw new ArgumentNullException("syntax");
            }

            _writer = writer;

            _fieldDelimiter = syntax.FieldDelimiter.ToString();
            _lineDelimiter  = string.Concat(syntax.LineDelimiter1, syntax.LineDelimiter2);
            _quote          = syntax.Quote.ToString();
            _escape         = syntax.Escape.ToString();

            _escapeAndQuote  = string.Concat(_escape, _quote);
            _escapeAndEscape = string.Concat(_escape, _escape);

            _hasHeaders = syntax.HasHeaders;
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
            }

            if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
            }

            if (this.Source.FieldDelimiter.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
            }

            if (this.Source.Quote != null && this.Source.Quote.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
            }

            if (this.Source.Escape != null && this.Source.Escape.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount = 0L;
            var validRowCount = 0L;
            var errorRowCount = 0L;

            var csvSyntax = new CsvSyntaxInfo
            {
                HasHeaders = this.Source.HasHeaders,
                FieldDelimiter = this.Source.FieldDelimiter[0],
                Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
                Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
                LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
                LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
            };

            using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
            {
                using (var csvReader = new CsvReader(fileReader, csvSyntax))
                {
                    var validator = new CsvValidator(this, csvReader, context, logger);
                    result.Status = validator.Validate();
                    sourceRowCount = validator.ReadRowCount;
                    validRowCount = validator.ValidRowCount;
                    errorRowCount = validator.ErrorRowCount;
                }
            }

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Found",
            //    Flags = sourceRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Validated",
            //    Flags = validRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Errors",
            //    Flags = errorRowCount,
            //});

            return result;
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
            }

            if (string.IsNullOrEmpty(this.Source.ConnectionString))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
            }

            if (string.IsNullOrEmpty(this.Source.ProviderName))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.ProviderName"));
            }

            if (string.IsNullOrEmpty(this.Source.Text))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.Text"));
            }

            if (string.IsNullOrEmpty(this.Destination.FieldDelimiter))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FieldDelimiter"));
            }

            if (string.IsNullOrEmpty(this.Destination.FilePath))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.FilePath"));
            }

            if (this.Destination.FieldDelimiter.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.FieldDelimiter", 1));
            }

            if (this.Destination.Quote != null && this.Destination.Quote.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Quote", 1));
            }

            if (this.Destination.Escape != null && this.Destination.Escape.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Destination.Escape", 1));
            }

            if (this.Destination != null)
            {
                var destinationFolderPath = Path.GetDirectoryName(this.Destination.FilePath);

                if (!string.IsNullOrEmpty(destinationFolderPath))
                {
                    if (!Directory.Exists(destinationFolderPath))
                    {
                        Directory.CreateDirectory(destinationFolderPath);
                    }
                }
            }

            //if (this.Mappings.Count == 0)
            //{
            //    throw new InvalidOperationException("Mappings not specified");
            //}

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount = 0L;
            var writtenRowCount = 0L;

            var encoding = Encoding.GetEncoding(this.Destination.CodePage);
            var csvSyntax = new CsvSyntaxInfo
            {
                HasHeaders = this.Destination.HasHeaders,
                FieldDelimiter = this.Destination.FieldDelimiter[0],
                Quote = this.Destination.Quote != null ? this.Destination.Quote[0] : DefaultQuote,
                Escape = this.Destination.Escape != null ? this.Destination.Escape[0] : DefaultEscape,
                LineDelimiter1 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[0] : DefaultLineDelimiter[0],
                LineDelimiter2 = this.Destination.LineDelimiter != null ? this.Destination.LineDelimiter[1] : DefaultLineDelimiter[1],
            };

            using (var dbAccessor = new DBAccessor(this.Source.ConnectionString, this.Source.ProviderName))
            {
                using (var dbReader = dbAccessor.ExecuteQuery(this.Source.Text, EtlQueryParameter.ToDictionary(this.Source.Parameters), this.TimeoutMilliseconds))
                {
                    using (var mapReader = new EtlMappedDataReader(dbReader, this.Mappings))
                    {
                        using (var fileWriter = new StreamWriter(this.Destination.FilePath, false, encoding))
                        {
                            var csvWriter = new CsvWriter(fileWriter, csvSyntax);

                            writtenRowCount = csvWriter.Write(mapReader);
                            sourceRowCount = writtenRowCount;
                        }
                    }
                }
            }

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Found",
            //    Flags = sourceRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Exported",
            //    Flags = writtenRowCount,
            //});

            return result;
        }
        public override EtlStepResult Invoke(EtlContext context, IEtlLogger logger)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            if (logger == null)
            {
                throw new ArgumentNullException("logger");
            }

            if (this.Source == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source"));
            }

            if (this.Destination == null)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination"));
            }

            if (string.IsNullOrEmpty(this.Source.FieldDelimiter))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FieldDelimiter"));
            }

            if (string.IsNullOrEmpty(this.Source.FilePath))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Source.FilePath"));
            }

            if (string.IsNullOrEmpty(this.Destination.ConnectionString))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ConnectionString"));
            }

            if (string.IsNullOrEmpty(this.Destination.ProviderName))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.ProviderName"));
            }

            if (string.IsNullOrEmpty(this.Destination.TableName))
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyCannotBeNull, "Destination.TableName"));
            }

            if (this.Source.FieldDelimiter.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.FieldDelimiter", 1));
            }

            if (this.Source.Quote != null && this.Source.Quote.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Quote", 1));
            }

            if (this.Source.Escape != null && this.Source.Escape.Length != 1)
            {
                throw new InvalidOperationException(string.Format(Properties.Resources.PropertyTooLong, "Source.Escape", 1));
            }

            var result = new EtlStepResult(EtlStatus.Succeeded, null);

            var sourceRowCount = 0L;
            var insertedRowCount = 0L;
            var errorRowCount = 0L;

            var csvSyntax = new CsvSyntaxInfo
            {
                HasHeaders = this.Source.HasHeaders,
                FieldDelimiter = this.Source.FieldDelimiter[0],
                Quote = this.Source.Quote != null ? this.Source.Quote[0] : DefaultQuote,
                Escape = this.Source.Escape != null ? this.Source.Escape[0] : DefaultEscape,
                LineDelimiter1 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[0] : DefaultLineDelimiter[0],
                LineDelimiter2 = this.Source.LineDelimiter != null ? this.Source.LineDelimiter[1] : DefaultLineDelimiter[1],
            };

            using (var fileReader = new StreamReader(this.Source.FilePath, Encoding.GetEncoding(this.Source.CodePage)))
            {
                using (var csvReader = new CsvReader(fileReader, csvSyntax))
                {
                    var wrappedReader = new EtlMappedDataReader(csvReader, this.Mappings);

                    var dbWriter = new DBTableWriter(this.Destination.ConnectionString, this.Destination.ProviderName, this.Destination.TableName);
                    dbWriter.ErrorOccured += delegate(object sender, DBTableWriterErrorEventArgs e)
                    {
                        var errorDateTime = DateTime.Now;
                        result.Status = EtlStatus.FinishedWithLosses;
                        result.Message = e.Message;

                        //todo: fix error counter in CSV import. Now it counts errors, not error records
                        errorRowCount++;

                        logger.LogEtlMessage
                        (
                            new EtlMessage
                            {
                                EtlPackageId = context.EtlPackageId,
                                EtlSessionId = context.EtlSessionId,
                                EtlStepName = this.Name,
                                LogDateTime = errorDateTime,
                                LogUtcDateTime = errorDateTime.ToUniversalTime(),
                                MessageType = EtlMessageType.Error,
                                Text = e.Message,
                                Flags = e.RecordIndex,
                                StackTrace = e.Exception != null ? e.Exception.StackTrace : null,
                            }
                        );

                        if (this.DataLossBehavior == EtlImportDataLossBehavior.Skip)
                        {
                            result.Status = EtlStatus.FinishedWithLosses;
                            e.TrySkipError = true;
                        }
                        else
                        {
                            result.Status = EtlStatus.Failed;
                        }
                    };

                    insertedRowCount = dbWriter.Write(wrappedReader, this.TimeoutMilliseconds, this.BatchSize);
                    sourceRowCount = csvReader.CurrentRecordIndex + 1;
                }
            }

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Errors",
            //    Flags = errorRowCount,
            //});

            //logger.LogEtlMessage(new EtlMessage
            //{
            //    EtlPackageId = context.EtlPackageId,
            //    EtlSessionId = context.EtlSessionId,
            //    EtlStepId = this.Id,
            //    LogDateTime = endDateTime,
            //    LogUtcDateTime = endDateTime.ToUniversalTime(),
            //    MessageType = EtlMessageType.Statistics,
            //    Text = "Inserted",
            //    Flags = insertedRowCount,
            //});

            return result;
        }