| public ReadBlock ( char buffer, int index, int count ) : int | ||
| buffer | char | |
| index | int | |
| count | int | |
| return | int | |
public List<string> ParseText(TextReader pReader)
{
result.Clear();
int count = 0;
do
{
count = pReader.ReadBlock(buffer, 0, BUFFER_SIZE);
for (int i = 0; i < count; ++i)
{
char c = buffer[i];
if (char.IsWhiteSpace(c))
{
ParseWhitespace();
}
else if (Contains(punctuation, c))
{
ParsePunctuation(c);
}
else if (char.IsLetterOrDigit(c))
{
ParseLetter(c);
}else{
ParseLetter(c);
}
}
} while(count == BUFFER_SIZE);
return result;
}
public void Process(string source, string target, string config,
TextReader input, TextWriter output)
{
Reader reader =
delegate (IntPtr data)
{
int c = input.ReadBlock(chars, 0, charSize);
int b = encoding.GetBytes(chars, 0, c, bytes, 0);
Marshal.Copy(bytes, 0, data, b);
return b;
};
Writer writer =
delegate (IntPtr data, int length)
{
if (length > 0) {
Marshal.Copy(data, bytes, 0, length);
int c = encoding.GetChars(bytes, 0, length, chars, 0);
output.Write(chars, 0, c);
}
};
string err = Native.pandoc(byteSize,
Bytes(source),
Bytes(target),
Bytes(config),
reader,
writer);
if (err != null) {
throw new PandocException(err);
}
}
public CompilationUnit Parse(TextReader reader)
{
// TODO: can we optimize this to avoid the text->stream->text roundtrip?
using (MemoryStream stream = new MemoryStream ()) {
StreamWriter w = new StreamWriter(stream, Encoding.UTF8);
char[] buffer = new char[2048];
int read;
while ((read = reader.ReadBlock(buffer, 0, buffer.Length)) > 0)
w.Write(buffer, 0, read);
w.Flush(); // we can't close the StreamWriter because that would also close the MemoryStream
stream.Position = 0;
return Parse(stream);
}
}
public static string Texto(TextReader leitor)
{
char[] buffer = new char[140];
StringBuilder builder = new StringBuilder("");
leitor.ReadBlock(buffer, 0, 140); //lê um bloco de 140 caracteres e coloca no buffer
//foreach (var caracter in buffer)
//show = show + caracter.ToString();
foreach (char c in buffer) //transforma o char[] em um construtor de strings
builder.Append(c);
//if (char.IsLetterOrDigit(c) || char.IsWhiteSpace(c))
return builder.ToString(); //retorna a string lida
}
public virtual Text Parse(TextReader reader)
{
Text textResult = new Text();
char[] buffer = new char[Bufferlength];
StringBuilder builder = new StringBuilder(Bufferlength);
int readCharsCount = 0;
do
{
readCharsCount = reader.ReadBlock(buffer, 0, Bufferlength);
if (readCharsCount <= 0) continue;
builder.Append(buffer, 0, readCharsCount);
int endIndex;
textResult += ParseText(builder.ToString(), out endIndex);
builder.Remove(0, endIndex + 1);
} while (readCharsCount != 0);
return textResult;
}
private static string GetDiffTreeLine(TextReader reader)
{
var sb = new StringBuilder();
char[] block = new char[98];
var bytesRead = reader.ReadBlock(block, 0, 98);
if (bytesRead == 0)
{
return null;
}
else if (bytesRead != 98)
{
throw new Exception("Invalid input.");
}
var nullBytesLeft = 2;
if (block[97] == 'C' || block[97] == 'R')
{
nullBytesLeft = 3;
}
sb.Append(block);
while (nullBytesLeft > 0)
{
var currentByte = reader.Read();
if (currentByte == -1)
{
throw new Exception("Invalid input.");
}
else if (currentByte == 0)
{
nullBytesLeft--;
}
sb.Append((char)currentByte);
}
return sb.ToString();
}
public override int ReadBlock([In, Out] char[] buffer, int index, int count)
{
return(_in.ReadBlock(buffer, index, count));
}
public override int ReadBlock(char[] buffer, int index, int count) => _in.ReadBlock(buffer, index, count);
public List<object> ParseText(TextReader pReader)
{
result.Clear();
int count = 0;
do
{
count = pReader.ReadBlock(buffer, 0, BUFFER_SIZE);
for (int i = 0; i < count; i++)
{
char c = buffer[i];
if (escape)
{
switch (c)
{
case 't':
AddLetter('\t');
break;
case 'r':
AddLetter('\r');
break;
case 'n':
AddLetter('\n');
break;
case 'b':
AddLetter('\b');
break;
case 'f':
AddLetter('\f');
break;
case 'v':
AddLetter('\v');
break;
default:
AddLetter(c);
break;
}
escape = false;
}
else
{
switch (c)
{
case '{':
case '[':
case '}':
case ']':
case ':':
case ',': //Structural chars
if (insideQuotes)
{
AddLetter(c);
}
else
{
PushStringToken();
AddLetter(c);
PushStringToken();
}
break;
case '"': //Quotes, string start stop
if (insideQuotes)
{
if (escape)
{
AddLetter(c);
}
else
{
PushStringDelimiter();
insideQuotes = false;
}
}
else
{
PushStringDelimiter();
insideQuotes = true;
}
break;
case '\'': //Whitespace and garbage
case '\f':
case '\v':
case '\b':
case '\r':
case '\n':
case ' ':
case '\t':
if (insideQuotes)
{
AddLetter(c);
}
break;
case '\\':
if (insideQuotes)
{
//don't add this letter, only escape next
escape = true;
}
break;
default:
AddLetter(c);
break;
}
}
}
} while(count == BUFFER_SIZE);
PushStringToken();
return result;
}
public override int ReadBlock(char [] buffer, int index, int count)
{
lock (this){
return(reader.ReadBlock(buffer, index, count));
}
}
///<summary>
///根据context的上下文情况，填充segmentBuff
/// </summary>
/// <returns>返回待分析的（有效的）字串长度</returns>
private int FillBuffer(TextReader reader)
{
int readCount = 0;
if (context.BuffOffset == 0)
{
//首次读取reader
readCount = reader.ReadBlock(segmentBuff, 0, segmentBuff.Length);
}
else
{
int offset = context.Available - context.LastAnalyzed;
if (offset > 0)
{
//最近一次读取的>最近一次处理的，将未处理的字串拷贝到segmentBuff头部
Array.Copy(segmentBuff, context.LastAnalyzed, this.segmentBuff, 0, offset);
readCount = offset;
}
//继续读取reader ，以onceReadIn - onceAnalyzed为起始位置，继续填充segmentBuff剩余的部分
readCount += reader.Read(segmentBuff, offset, BUFF_SIZE - offset);
}
//记录最后一次从Reader中读入的可用字符长度
context.Available = readCount;
return readCount;
}
private static string ReadString(TextReader reader)
{
reader.Read(); // read '"'
StringBuilder builder = new StringBuilder();
bool isControl = false;
while (reader.Peek()>=0)
{
char c = (char)reader.Read();
if (isControl)
{
switch (c)
{
case '"':
builder.Append('"');
break;
case '\\':
builder.Append('\\');
break;
case '/':
builder.Append('/');
break;
case 'b':
builder.Append('\b');
break;
case 'f':
builder.Append('\f');
break;
case 'n':
builder.Append('\n');
break;
case 'r':
builder.Append('\r');
break;
case 't':
builder.Append('\t');
break;
case 'u':
{
if (reader.ReadBlock(hexBuffer, 0, 4) < 4)
throw new IOException("Invalid JSON format.");
string hexString = new string(hexBuffer);
long result = long.Parse(hexString, System.Globalization.NumberStyles.HexNumber);
char hexChar = (char)result;
builder.Append(hexChar);
}
break;
default:
throw new IOException("Invalid JSON format.");
}
isControl = false;
}
else
{
if (c == '"')
break;
else if (c == '\\')
{
isControl = true;
}
else
{
builder.Append(c);
}
}
}
return builder.ToString();
}
/// <summary>
/// Read an escaped character e.g. "\n". Must be performed after the backslash has been read.
/// </summary>
/// <param name="reader">The text reader.</param>
/// <returns>The escaped character.</returns>
private char ReadEscapedCharacter(TextReader reader)
{
char c;
if (!reader.TryRead(out c))
throw new Exception("Expected escape character");
switch (c) {
case '0':
return '\0';
case 't':
return '\t';
case 'r':
return '\r';
case 'n':
return '\n';
case 'x':
char[] hexCode = new char[4];
if (reader.ReadBlock(hexCode, 0, 4) != 4)
throw new Exception("Wrong format for escaped unicode character.");
return (char)Int32.Parse(new string(hexCode), NumberStyles.AllowHexSpecifier);
default:
return c;
}
}
public static void Preprocess(TextReader reader, TextWriter writer, IReadOnlyDictionary<string, string> values)
{
// $$ is the escape for a $, which we can just replicate in the algorithm by giving the empty replacement the known value
var maximumKeyLength = values.Keys.Max<string, int?>(k => k.Length).GetValueOrDefault(0);
char[] buffer = new char[Math.Max(4096, maximumKeyLength + 2)];
int charsInBuffer = reader.ReadBlock(buffer, 0, buffer.Length);
while (charsInBuffer != 0)
{
int indexOfDelimiter = Array.IndexOf(buffer, PreprocessorDelimiter, 0, charsInBuffer);
if (indexOfDelimiter == -1)
{
// If the buffer doesn't contain any delimiters, then we can immediately write it out and move on
writer.Write(buffer, 0, charsInBuffer);
charsInBuffer = reader.ReadBlock(buffer, 0, buffer.Length);
}
else
{
// Write whatever is before the $ and advance up to the $
writer.Write(buffer, 0, indexOfDelimiter);
Advance(reader, buffer, ref charsInBuffer, charsToAdvance: indexOfDelimiter);
// Let's read in the token name
var token = new StringBuilder();
int position = 1;
while (true)
{
if (position == buffer.Length)
{
Advance(reader, buffer, ref charsInBuffer, buffer.Length);
position = 0;
}
// If at end, we'll want to fall through to the last case
var c = position < charsInBuffer ? buffer[position] : '\0';
if (c == PreprocessorDelimiter)
{
position++;
// Is it the escape case?
string value;
if (token.Length == 0)
{
writer.Write(PreprocessorDelimiter);
}
else if (values.TryGetValue(token.ToString(), out value))
{
writer.Write(value);
}
else
{
throw new ExceptionFromResource(nameof(Strings.UnspecifiedToken), PreprocessorDelimiter + token.ToString() + PreprocessorDelimiter);
}
break;
}
else if (IsTokenCharacter(c))
{
token.Append(c);
position++;
}
else
{
// The token ended prematurely, so we just treat it verbatim and write it out
writer.Write(PreprocessorDelimiter);
writer.Write(token);
break;
}
}
// Advance to the next position to start all over
Advance(reader, buffer, ref charsInBuffer, position);
}
}
}
private static void Advance(TextReader reader, char[] buffer, ref int charsInBuffer, int charsToAdvance)
{
Debug.Assert(charsToAdvance <= charsInBuffer);
// Move the remaining characters in the buffer forward
Array.Copy(sourceArray: buffer, sourceIndex: charsToAdvance, destinationArray: buffer, destinationIndex: 0, length: charsInBuffer - charsToAdvance);
charsInBuffer -= charsToAdvance;
charsInBuffer += reader.ReadBlock(buffer, charsInBuffer, buffer.Length - charsInBuffer);
}
private static ImmutableArray<char[]> ReadChunksFromTextReader(TextReader reader, int maxCharRemainingGuess, bool throwIfBinaryDetected)
{
var chunks = ArrayBuilder<char[]>.GetInstance(1 + maxCharRemainingGuess / ChunkSize);
while (reader.Peek() != -1)
{
var nextChunkSize = ChunkSize;
if (maxCharRemainingGuess < ChunkSize)
{
// maxCharRemainingGuess typically overestimates a little
// so we will first fill a slightly smaller (maxCharRemainingGuess - 64) chunk
// and then use 64 char tail, which is likley to be resized.
nextChunkSize = Math.Max(maxCharRemainingGuess - 64, 64);
}
char[] chunk = new char[nextChunkSize];
int charsRead = reader.ReadBlock(chunk, 0, chunk.Length);
if (charsRead == 0)
{
break;
}
maxCharRemainingGuess -= charsRead;
if (charsRead < chunk.Length)
{
Array.Resize(ref chunk, charsRead);
}
// Check for binary files
if (throwIfBinaryDetected && IsBinary(chunk))
{
throw new InvalidDataException();
}
chunks.Add(chunk);
}
return chunks.ToImmutableAndFree();
}
