public ITextFileParser GetSchemaParser(FileProfile fileProfile, Stream stream) { if (fileProfile.IsFixedWidth) { var parser = new FixedWidthTextFileParser(stream); if (fileProfile.SchemaDetector.ParseStartPosition.HasValue && fileProfile.SchemaDetector.ParseStartPosition.Value > 0 && fileProfile.SchemaDetector.ParseLength.HasValue && fileProfile.SchemaDetector.ParseLength.Value > 0) { parser.AddFieldWidth(fileProfile.SchemaDetector.ParseStartPosition.Value, fileProfile.SchemaDetector.ParseLength.Value); fileProfile.SchemaDetector.SourceColumnNumber = 1; } return(parser); } else { var parser = new DelimitedTextFileParser(stream); parser.Delimiters = new[] { fileProfile.Delimiter }; if (!string.IsNullOrWhiteSpace(fileProfile.TextQualifier)) { parser.HasFieldsEnclosedInQualifier = true; parser.SetTextQualifier(fileProfile.TextQualifier); } else { parser.HasFieldsEnclosedInQualifier = false; } return(parser); } }
public ITextFileParser GetParser(FileProfile fileProfile, Stream stream) { if (fileProfile.IsFixedWidth) { var parser = new FixedWidthTextFileParser(stream); int sourceColNum = 1; foreach (RowDefinition rowDefinition in fileProfile.FileTypes[0].RowDefinitions) { if (rowDefinition.ParseStartPosition.HasValue && rowDefinition.ParseStartPosition.Value > 0 && rowDefinition.ParseLength.HasValue && rowDefinition.ParseLength.Value > 0) { parser.AddFieldWidth( rowDefinition.ParseStartPosition.Value, rowDefinition.ParseLength.Value); rowDefinition.SourceColumnNumber = sourceColNum; sourceColNum++; } else { rowDefinition.SourceColumnNumber = null; } } return(parser); } else { var parser = new DelimitedTextFileParser(stream); parser.Delimiters = new[] { fileProfile.Delimiter }; if (!string.IsNullOrWhiteSpace(fileProfile.TextQualifier)) { parser.HasFieldsEnclosedInQualifier = true; parser.SetTextQualifier(fileProfile.TextQualifier); } else { parser.HasFieldsEnclosedInQualifier = false; } return(parser); } }
private void SplitSchemas() { if (!_isEdiFileFormat) { long totalLines = _fileData.GetLineCount(); _result.TotalLines = totalLines; if (!_fileToProcess.IsFixedWidth && (_fileToProcess.HeaderLines.HasValue && _fileToProcess.HeaderLines > 0) && _fileToProcess.CaptureUnmappedDataForInboundFile) { Stream headerStream = null; DelimitedTextFileParser headerParser = null; try { headerStream = _fileData.GetReadStream(); headerParser = new DelimitedTextFileParser(headerStream); headerParser.SetDelimiters(_fileToProcess.Delimiter); headerParser.SetTextQualifier(_fileToProcess.TextQualifier); _headerFields = headerParser.ReadFields(); //BK : This code is doing nothing and spending cycle just finding byteread which is never used. //this takes around 15 minutes with big file. //int byteRead = 0; //while (byteRead != -1) //{ // byteRead = headerStream.ReadByte(); //} } catch { //swallow, ignore headers _headerFields = null; } finally { if (headerStream != null) { try { headerStream.Dispose(); } catch { //swallow errors //hopefully the stream eventually gets disposed by GC } } if (headerParser != null) { headerParser.Dispose(); } } } if (!_fileToProcess.IsMultiSchema) { _result.TemporaryFiles.Add("Default", _fileData); return; } long footerLines = _fileToProcess.FooterLines.HasValue ? _fileToProcess.FooterLines.Value : 0; StreamReader readReader = _fileData.GetStreamReader(); StreamReader parserReader = _fileData.GetStreamReader(); ITextFileParser schemaParser = ParserFactory.Instance.GetSchemaParser <ITextFileParser>(_fileToProcess, parserReader.BaseStream); SkipHeaderLines(readReader, schemaParser); while (!schemaParser.EndOfData) { _currentLineNumber++; if (footerLines > 0 && _currentLineNumber > totalLines - footerLines) { break; } string[] fields = schemaParser.ReadFields(); if (fields == null) { _result.BlankLines++; readReader.ReadLine(); continue; } RowDefinition rowDef = _fileToProcess.SchemaDetector; IContext context = new Context((DataTypeEnum)rowDef.DataType, rowDef.IsRequired, rowDef.CanSetDefault, rowDef.DefaultValue, _fileToProcess.FileTypes[0].Description, (_fileToProcess.IsFixedWidth ? null : rowDef.ParseStartPosition), (_fileToProcess.IsFixedWidth ? null : rowDef.ParseLength), rowDef.FieldFormat, rowDef.FieldDisplayName, rowDef.FieldDescription, rowDef.TargetTableName, rowDef.TargetFieldName); IColumn column = (_fileToProcess.IsFixedWidth ? new Column(fields[0], context) : new Column(fields[rowDef.SourceColumnNumber.Value - 1], context)); //TODO: CryptoStream seems to convert blank lines into a bunch of escaped nulls //is there a better way to detect this condition, or prevent it all together if (column.ActualValue != null && column.ActualValue.ToString() != "\0\0") { var currentWriter = GetStreamWriter(column.ActualValue); string line = readReader.ReadLine(); int originalLineLength = line.Length; const int rowNumLength = 10; if (_fileToProcess.IsFixedWidth) { line += _currentLineNumber.ToString().ToFixedLength(rowNumLength, ' ', true); } else { line += (_fileToProcess.Delimiter.ToLower().Trim() == "tab" ? "\t" : _fileToProcess.Delimiter) + _currentLineNumber.ToString(); } currentWriter.WriteLine(line); } } CloseWriterStreams(); } else { _result.TemporaryFiles.Add("EdiDefault", _fileData); } }