public ITextFileParser GetSchemaParser(FileProfile fileProfile, Stream stream)
 {
     if (fileProfile.IsFixedWidth)
     {
         var parser = new FixedWidthTextFileParser(stream);
         if (fileProfile.SchemaDetector.ParseStartPosition.HasValue && fileProfile.SchemaDetector.ParseStartPosition.Value > 0 &&
             fileProfile.SchemaDetector.ParseLength.HasValue && fileProfile.SchemaDetector.ParseLength.Value > 0)
         {
             parser.AddFieldWidth(fileProfile.SchemaDetector.ParseStartPosition.Value, fileProfile.SchemaDetector.ParseLength.Value);
             fileProfile.SchemaDetector.SourceColumnNumber = 1;
         }
         return(parser);
     }
     else
     {
         var parser = new DelimitedTextFileParser(stream);
         parser.Delimiters = new[] { fileProfile.Delimiter };
         if (!string.IsNullOrWhiteSpace(fileProfile.TextQualifier))
         {
             parser.HasFieldsEnclosedInQualifier = true;
             parser.SetTextQualifier(fileProfile.TextQualifier);
         }
         else
         {
             parser.HasFieldsEnclosedInQualifier = false;
         }
         return(parser);
     }
 }
 public ITextFileParser GetParser(FileProfile fileProfile, Stream stream)
 {
     if (fileProfile.IsFixedWidth)
     {
         var parser       = new FixedWidthTextFileParser(stream);
         int sourceColNum = 1;
         foreach (RowDefinition rowDefinition in fileProfile.FileTypes[0].RowDefinitions)
         {
             if (rowDefinition.ParseStartPosition.HasValue && rowDefinition.ParseStartPosition.Value > 0 &&
                 rowDefinition.ParseLength.HasValue && rowDefinition.ParseLength.Value > 0)
             {
                 parser.AddFieldWidth(
                     rowDefinition.ParseStartPosition.Value,
                     rowDefinition.ParseLength.Value);
                 rowDefinition.SourceColumnNumber = sourceColNum;
                 sourceColNum++;
             }
             else
             {
                 rowDefinition.SourceColumnNumber = null;
             }
         }
         return(parser);
     }
     else
     {
         var parser = new DelimitedTextFileParser(stream);
         parser.Delimiters = new[] { fileProfile.Delimiter };
         if (!string.IsNullOrWhiteSpace(fileProfile.TextQualifier))
         {
             parser.HasFieldsEnclosedInQualifier = true;
             parser.SetTextQualifier(fileProfile.TextQualifier);
         }
         else
         {
             parser.HasFieldsEnclosedInQualifier = false;
         }
         return(parser);
     }
 }
Пример #3
0
        private void SplitSchemas()
        {
            if (!_isEdiFileFormat)
            {
                long totalLines = _fileData.GetLineCount();
                _result.TotalLines = totalLines;

                if (!_fileToProcess.IsFixedWidth && (_fileToProcess.HeaderLines.HasValue && _fileToProcess.HeaderLines > 0) && _fileToProcess.CaptureUnmappedDataForInboundFile)
                {
                    Stream headerStream = null;
                    DelimitedTextFileParser headerParser = null;
                    try
                    {
                        headerStream = _fileData.GetReadStream();
                        headerParser = new DelimitedTextFileParser(headerStream);
                        headerParser.SetDelimiters(_fileToProcess.Delimiter);
                        headerParser.SetTextQualifier(_fileToProcess.TextQualifier);
                        _headerFields = headerParser.ReadFields();

                        //BK : This code is doing nothing and spending cycle just finding byteread which is never used.
                        //this takes around 15 minutes with big file.
                        //int byteRead = 0;
                        //while (byteRead != -1)
                        //{
                        //    byteRead = headerStream.ReadByte();
                        //}
                    }
                    catch
                    {
                        //swallow, ignore headers
                        _headerFields = null;
                    }
                    finally
                    {
                        if (headerStream != null)
                        {
                            try
                            {
                                headerStream.Dispose();
                            }
                            catch
                            {
                                //swallow errors
                                //hopefully the stream eventually gets disposed by GC
                            }
                        }
                        if (headerParser != null)
                        {
                            headerParser.Dispose();
                        }
                    }
                }
                if (!_fileToProcess.IsMultiSchema)
                {
                    _result.TemporaryFiles.Add("Default", _fileData);
                    return;
                }

                long         footerLines  = _fileToProcess.FooterLines.HasValue ? _fileToProcess.FooterLines.Value : 0;
                StreamReader readReader   = _fileData.GetStreamReader();
                StreamReader parserReader = _fileData.GetStreamReader();

                ITextFileParser schemaParser = ParserFactory.Instance.GetSchemaParser <ITextFileParser>(_fileToProcess, parserReader.BaseStream);

                SkipHeaderLines(readReader, schemaParser);

                while (!schemaParser.EndOfData)
                {
                    _currentLineNumber++;
                    if (footerLines > 0 && _currentLineNumber > totalLines - footerLines)
                    {
                        break;
                    }

                    string[] fields = schemaParser.ReadFields();
                    if (fields == null)
                    {
                        _result.BlankLines++;
                        readReader.ReadLine();
                        continue;
                    }

                    RowDefinition rowDef  = _fileToProcess.SchemaDetector;
                    IContext      context = new Context((DataTypeEnum)rowDef.DataType, rowDef.IsRequired, rowDef.CanSetDefault,
                                                        rowDef.DefaultValue, _fileToProcess.FileTypes[0].Description,
                                                        (_fileToProcess.IsFixedWidth ? null : rowDef.ParseStartPosition),
                                                        (_fileToProcess.IsFixedWidth ? null : rowDef.ParseLength), rowDef.FieldFormat,
                                                        rowDef.FieldDisplayName, rowDef.FieldDescription, rowDef.TargetTableName,
                                                        rowDef.TargetFieldName);

                    IColumn column = (_fileToProcess.IsFixedWidth ? new Column(fields[0], context) : new Column(fields[rowDef.SourceColumnNumber.Value - 1], context));

                    //TODO: CryptoStream seems to convert blank lines into a bunch of escaped nulls
                    //is there a better way to detect this condition, or prevent it all together
                    if (column.ActualValue != null && column.ActualValue.ToString() != "\0\0")
                    {
                        var       currentWriter      = GetStreamWriter(column.ActualValue);
                        string    line               = readReader.ReadLine();
                        int       originalLineLength = line.Length;
                        const int rowNumLength       = 10;
                        if (_fileToProcess.IsFixedWidth)
                        {
                            line += _currentLineNumber.ToString().ToFixedLength(rowNumLength, ' ', true);
                        }
                        else
                        {
                            line += (_fileToProcess.Delimiter.ToLower().Trim() == "tab" ? "\t" : _fileToProcess.Delimiter) + _currentLineNumber.ToString();
                        }
                        currentWriter.WriteLine(line);
                    }
                }

                CloseWriterStreams();
            }
            else
            {
                _result.TemporaryFiles.Add("EdiDefault", _fileData);
            }
        }