public List<string> ParseText(TextReader pReader) { result.Clear(); int count = 0; do { count = pReader.ReadBlock(buffer, 0, BUFFER_SIZE); for (int i = 0; i < count; ++i) { char c = buffer[i]; if (char.IsWhiteSpace(c)) { ParseWhitespace(); } else if (Contains(punctuation, c)) { ParsePunctuation(c); } else if (char.IsLetterOrDigit(c)) { ParseLetter(c); }else{ ParseLetter(c); } } } while(count == BUFFER_SIZE); return result; }
public void Process(string source, string target, string config, TextReader input, TextWriter output) { Reader reader = delegate (IntPtr data) { int c = input.ReadBlock(chars, 0, charSize); int b = encoding.GetBytes(chars, 0, c, bytes, 0); Marshal.Copy(bytes, 0, data, b); return b; }; Writer writer = delegate (IntPtr data, int length) { if (length > 0) { Marshal.Copy(data, bytes, 0, length); int c = encoding.GetChars(bytes, 0, length, chars, 0); output.Write(chars, 0, c); } }; string err = Native.pandoc(byteSize, Bytes(source), Bytes(target), Bytes(config), reader, writer); if (err != null) { throw new PandocException(err); } }
public CompilationUnit Parse(TextReader reader) { // TODO: can we optimize this to avoid the text->stream->text roundtrip? using (MemoryStream stream = new MemoryStream ()) { StreamWriter w = new StreamWriter(stream, Encoding.UTF8); char[] buffer = new char[2048]; int read; while ((read = reader.ReadBlock(buffer, 0, buffer.Length)) > 0) w.Write(buffer, 0, read); w.Flush(); // we can't close the StreamWriter because that would also close the MemoryStream stream.Position = 0; return Parse(stream); } }
public static string Texto(TextReader leitor) { char[] buffer = new char[140]; StringBuilder builder = new StringBuilder(""); leitor.ReadBlock(buffer, 0, 140); //lê um bloco de 140 caracteres e coloca no buffer //foreach (var caracter in buffer) //show = show + caracter.ToString(); foreach (char c in buffer) //transforma o char[] em um construtor de strings builder.Append(c); //if (char.IsLetterOrDigit(c) || char.IsWhiteSpace(c)) return builder.ToString(); //retorna a string lida }
public virtual Text Parse(TextReader reader) { Text textResult = new Text(); char[] buffer = new char[Bufferlength]; StringBuilder builder = new StringBuilder(Bufferlength); int readCharsCount = 0; do { readCharsCount = reader.ReadBlock(buffer, 0, Bufferlength); if (readCharsCount <= 0) continue; builder.Append(buffer, 0, readCharsCount); int endIndex; textResult += ParseText(builder.ToString(), out endIndex); builder.Remove(0, endIndex + 1); } while (readCharsCount != 0); return textResult; }
private static string GetDiffTreeLine(TextReader reader) { var sb = new StringBuilder(); char[] block = new char[98]; var bytesRead = reader.ReadBlock(block, 0, 98); if (bytesRead == 0) { return null; } else if (bytesRead != 98) { throw new Exception("Invalid input."); } var nullBytesLeft = 2; if (block[97] == 'C' || block[97] == 'R') { nullBytesLeft = 3; } sb.Append(block); while (nullBytesLeft > 0) { var currentByte = reader.Read(); if (currentByte == -1) { throw new Exception("Invalid input."); } else if (currentByte == 0) { nullBytesLeft--; } sb.Append((char)currentByte); } return sb.ToString(); }
public override int ReadBlock([In, Out] char[] buffer, int index, int count) { return(_in.ReadBlock(buffer, index, count)); }
public override int ReadBlock(char[] buffer, int index, int count) => _in.ReadBlock(buffer, index, count);
public List<object> ParseText(TextReader pReader) { result.Clear(); int count = 0; do { count = pReader.ReadBlock(buffer, 0, BUFFER_SIZE); for (int i = 0; i < count; i++) { char c = buffer[i]; if (escape) { switch (c) { case 't': AddLetter('\t'); break; case 'r': AddLetter('\r'); break; case 'n': AddLetter('\n'); break; case 'b': AddLetter('\b'); break; case 'f': AddLetter('\f'); break; case 'v': AddLetter('\v'); break; default: AddLetter(c); break; } escape = false; } else { switch (c) { case '{': case '[': case '}': case ']': case ':': case ',': //Structural chars if (insideQuotes) { AddLetter(c); } else { PushStringToken(); AddLetter(c); PushStringToken(); } break; case '"': //Quotes, string start stop if (insideQuotes) { if (escape) { AddLetter(c); } else { PushStringDelimiter(); insideQuotes = false; } } else { PushStringDelimiter(); insideQuotes = true; } break; case '\'': //Whitespace and garbage case '\f': case '\v': case '\b': case '\r': case '\n': case ' ': case '\t': if (insideQuotes) { AddLetter(c); } break; case '\\': if (insideQuotes) { //don't add this letter, only escape next escape = true; } break; default: AddLetter(c); break; } } } } while(count == BUFFER_SIZE); PushStringToken(); return result; }
public override int ReadBlock(char [] buffer, int index, int count) { lock (this){ return(reader.ReadBlock(buffer, index, count)); } }
///<summary> ///根据context的上下文情况,填充segmentBuff /// </summary> /// <returns>返回待分析的(有效的)字串长度</returns> private int FillBuffer(TextReader reader) { int readCount = 0; if (context.BuffOffset == 0) { //首次读取reader readCount = reader.ReadBlock(segmentBuff, 0, segmentBuff.Length); } else { int offset = context.Available - context.LastAnalyzed; if (offset > 0) { //最近一次读取的>最近一次处理的,将未处理的字串拷贝到segmentBuff头部 Array.Copy(segmentBuff, context.LastAnalyzed, this.segmentBuff, 0, offset); readCount = offset; } //继续读取reader ,以onceReadIn - onceAnalyzed为起始位置,继续填充segmentBuff剩余的部分 readCount += reader.Read(segmentBuff, offset, BUFF_SIZE - offset); } //记录最后一次从Reader中读入的可用字符长度 context.Available = readCount; return readCount; }
private static string ReadString(TextReader reader) { reader.Read(); // read '"' StringBuilder builder = new StringBuilder(); bool isControl = false; while (reader.Peek()>=0) { char c = (char)reader.Read(); if (isControl) { switch (c) { case '"': builder.Append('"'); break; case '\\': builder.Append('\\'); break; case '/': builder.Append('/'); break; case 'b': builder.Append('\b'); break; case 'f': builder.Append('\f'); break; case 'n': builder.Append('\n'); break; case 'r': builder.Append('\r'); break; case 't': builder.Append('\t'); break; case 'u': { if (reader.ReadBlock(hexBuffer, 0, 4) < 4) throw new IOException("Invalid JSON format."); string hexString = new string(hexBuffer); long result = long.Parse(hexString, System.Globalization.NumberStyles.HexNumber); char hexChar = (char)result; builder.Append(hexChar); } break; default: throw new IOException("Invalid JSON format."); } isControl = false; } else { if (c == '"') break; else if (c == '\\') { isControl = true; } else { builder.Append(c); } } } return builder.ToString(); }
/// <summary> /// Read an escaped character e.g. "\n". Must be performed after the backslash has been read. /// </summary> /// <param name="reader">The text reader.</param> /// <returns>The escaped character.</returns> private char ReadEscapedCharacter(TextReader reader) { char c; if (!reader.TryRead(out c)) throw new Exception("Expected escape character"); switch (c) { case '0': return '\0'; case 't': return '\t'; case 'r': return '\r'; case 'n': return '\n'; case 'x': char[] hexCode = new char[4]; if (reader.ReadBlock(hexCode, 0, 4) != 4) throw new Exception("Wrong format for escaped unicode character."); return (char)Int32.Parse(new string(hexCode), NumberStyles.AllowHexSpecifier); default: return c; } }
public static void Preprocess(TextReader reader, TextWriter writer, IReadOnlyDictionary<string, string> values) { // $$ is the escape for a $, which we can just replicate in the algorithm by giving the empty replacement the known value var maximumKeyLength = values.Keys.Max<string, int?>(k => k.Length).GetValueOrDefault(0); char[] buffer = new char[Math.Max(4096, maximumKeyLength + 2)]; int charsInBuffer = reader.ReadBlock(buffer, 0, buffer.Length); while (charsInBuffer != 0) { int indexOfDelimiter = Array.IndexOf(buffer, PreprocessorDelimiter, 0, charsInBuffer); if (indexOfDelimiter == -1) { // If the buffer doesn't contain any delimiters, then we can immediately write it out and move on writer.Write(buffer, 0, charsInBuffer); charsInBuffer = reader.ReadBlock(buffer, 0, buffer.Length); } else { // Write whatever is before the $ and advance up to the $ writer.Write(buffer, 0, indexOfDelimiter); Advance(reader, buffer, ref charsInBuffer, charsToAdvance: indexOfDelimiter); // Let's read in the token name var token = new StringBuilder(); int position = 1; while (true) { if (position == buffer.Length) { Advance(reader, buffer, ref charsInBuffer, buffer.Length); position = 0; } // If at end, we'll want to fall through to the last case var c = position < charsInBuffer ? buffer[position] : '\0'; if (c == PreprocessorDelimiter) { position++; // Is it the escape case? string value; if (token.Length == 0) { writer.Write(PreprocessorDelimiter); } else if (values.TryGetValue(token.ToString(), out value)) { writer.Write(value); } else { throw new ExceptionFromResource(nameof(Strings.UnspecifiedToken), PreprocessorDelimiter + token.ToString() + PreprocessorDelimiter); } break; } else if (IsTokenCharacter(c)) { token.Append(c); position++; } else { // The token ended prematurely, so we just treat it verbatim and write it out writer.Write(PreprocessorDelimiter); writer.Write(token); break; } } // Advance to the next position to start all over Advance(reader, buffer, ref charsInBuffer, position); } } }
private static void Advance(TextReader reader, char[] buffer, ref int charsInBuffer, int charsToAdvance) { Debug.Assert(charsToAdvance <= charsInBuffer); // Move the remaining characters in the buffer forward Array.Copy(sourceArray: buffer, sourceIndex: charsToAdvance, destinationArray: buffer, destinationIndex: 0, length: charsInBuffer - charsToAdvance); charsInBuffer -= charsToAdvance; charsInBuffer += reader.ReadBlock(buffer, charsInBuffer, buffer.Length - charsInBuffer); }
private static ImmutableArray<char[]> ReadChunksFromTextReader(TextReader reader, int maxCharRemainingGuess, bool throwIfBinaryDetected) { var chunks = ArrayBuilder<char[]>.GetInstance(1 + maxCharRemainingGuess / ChunkSize); while (reader.Peek() != -1) { var nextChunkSize = ChunkSize; if (maxCharRemainingGuess < ChunkSize) { // maxCharRemainingGuess typically overestimates a little // so we will first fill a slightly smaller (maxCharRemainingGuess - 64) chunk // and then use 64 char tail, which is likley to be resized. nextChunkSize = Math.Max(maxCharRemainingGuess - 64, 64); } char[] chunk = new char[nextChunkSize]; int charsRead = reader.ReadBlock(chunk, 0, chunk.Length); if (charsRead == 0) { break; } maxCharRemainingGuess -= charsRead; if (charsRead < chunk.Length) { Array.Resize(ref chunk, charsRead); } // Check for binary files if (throwIfBinaryDetected && IsBinary(chunk)) { throw new InvalidDataException(); } chunks.Add(chunk); } return chunks.ToImmutableAndFree(); }