public static string CalculateIndent(TextReader code, int line, bool tabsToSpaces = false, int indentWidth = 4)
{
if(line < 2)
return string.Empty;
var eng = new IndentEngine(DFormattingOptions.CreateDStandard(), tabsToSpaces, indentWidth);
int curLine = 1;
const int lf = (int)'\n';
const int cr = (int)'\r';
int c;
while((c = code.Read()) != -1)
{
if(c == lf || c == cr)
{
if(c == cr && code.Peek() == lf)
code.Read();
if(++curLine > line)
break;
}
eng.Push((char)c);
}
return eng.ThisLineIndent;
}
public static bool? ParseNullable(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
reader.Read();
return cur == 't';
}
public static bool Parse(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return false;
reader.Read();
return cur == 't';
}
public static DateTime Parse(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return DateTime.MinValue;
var res = ParseDate(reader, cur);
reader.Read();
return res;
}
public static DateTime? ParseNullable(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
var res = ParseDate(reader, cur);
reader.Read();
return res;
}
public static void CorrectIndent(TextReader code, int startOffset, int endOffset, Action<int, int, string> documentReplace, DFormattingOptions options = null, ITextEditorOptions textStyle = null, bool formatLastLine = true)
{
textStyle = textStyle ?? TextEditorOptions.Default;
var eng = new IndentEngine(options ?? DFormattingOptions.CreateDStandard(), textStyle.TabsToSpaces, textStyle.IndentSize, textStyle.KeepAlignmentSpaces);
var replaceActions = new List<DFormattingVisitor.TextReplaceAction>();
int originalIndent = 0;
bool hadLineBreak = true;
int n = 0;
for (int i = 0; i <= endOffset && (n = code.Read()) != -1; i++)
{
if(n == '\r' || n == '\n')
{
if (i >= startOffset && !eng.LineBeganInsideString)
replaceActions.Add(new DFormattingVisitor.TextReplaceAction(eng.Position - eng.LineOffset, originalIndent, eng.ThisLineIndent));
hadLineBreak = true;
originalIndent = 0;
if (code.Peek() == '\n')
{
eng.Push((char)code.Read());
i++;
}
}
else if(hadLineBreak)
{
if(n == ' ' || n== '\t')
originalIndent++;
else
hadLineBreak = false;
// If there's code left, format the last line of the selection either
if (i == endOffset && formatLastLine)
endOffset++;
}
eng.Push((char)n);
}
// Also indent the last line if we're at the EOF.
if (code.Peek() == -1 || (formatLastLine && n != '\r' && n != '\n'))
{
if(!eng.LineBeganInsideString)
replaceActions.Add(new DFormattingVisitor.TextReplaceAction(eng.Position - eng.LineOffset, originalIndent, eng.ThisLineIndent));
}
// Perform replacements from the back of the document to the front - to ensure offset consistency
for(int k = replaceActions.Count - 1; k>=0; k--)
{
var rep = replaceActions[k];
if(rep.RemovalLength > 0 || rep.NewText.Length != 0)
documentReplace(rep.Offset, rep.RemovalLength, rep.NewText);
}
}
protected void fillLookAheadBuffer()
{
try
{
lookAheadBuffer[0] = (char)input.Read();
}
catch (Exception e)
{
//e.printStackTrace();
}
}
private static object?ReadNull(System.IO.TextReader reader)
{
reader.EatWhiteSpace();
char ch = (char)reader.Peek();
if (ch == 'n')
{
reader.Read(); reader.Read(); reader.Read(); reader.Read(); // read chars n,u,l,l
}
return(null);
}
public static List<byte[]> ParseCollection(TextReader reader, int context, bool allowNulls)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
var espaced = cur != '{';
if (espaced)
{
for (int i = 0; i < context; i++)
reader.Read();
}
var innerContext = context << 1;
var skipInner = innerContext + (innerContext << 1);
var list = new List<byte[]>();
cur = reader.Peek();
if (cur == '}')
reader.Read();
var emptyColl = allowNulls ? null : new byte[0];
while (cur != -1 && cur != '}')
{
cur = reader.Read();
if (cur == 'N')
{
reader.Read();
reader.Read();
reader.Read();
list.Add(emptyColl);
cur = reader.Read();
}
else
{
for (int i = 0; i < skipInner; i++)
reader.Read();
var item = new List<byte>(1024);
cur = reader.Read();
while (cur != -1 && cur != '"' && cur != '\\')
{
item.Add((byte)((CharLookup[cur] << 4) + CharLookup[reader.Read()]));
cur = reader.Read();
}
for (int i = 0; i < innerContext; i++)
cur = reader.Read();
list.Add(item.ToArray());
}
}
if (espaced)
{
for (int i = 0; i < context; i++)
reader.Read();
}
reader.Read();
return list;
}
public static int ReadInt(TextReader reader)
{
int result = reader.Read() - '0';
var input = reader.Read();
while (input >= '0' && input <= '9')
{
result = (result * 10) + input - '0';
input = reader.Read();
}
return result;
}
public int Read(char[] c, int a, int b)
{
int res = rdr.Read(c, a, b);
if (annotate)
{
for (int i = 0; i < res; i++)
{
updpos(c[a + i]);
}
}
return(res);
}
internal new static JBoolean Parse(TextReader reader)
{
SkipSpace(reader);
char firstChar = (char)reader.Read();
if (firstChar == 't')
{
int c2 = reader.Read();
int c3 = reader.Read();
int c4 = reader.Read();
if (c2 == 'r' && c3 == 'u' && c4 == 'e')
{
return new JBoolean(true);
}
}
else if (firstChar == 'f')
{
int c2 = reader.Read();
int c3 = reader.Read();
int c4 = reader.Read();
int c5 = reader.Read();
if (c2 == 'a' && c3 == 'l' && c4 == 's' && c5 == 'e')
{
return new JBoolean(false);
}
}
throw new FormatException();
}
private char yy_advance()
{
int next_read;
int i;
int j;
if (yy_buffer_index < yy_buffer_read)
{
return(yy_buffer[yy_buffer_index++]);
}
if (0 != yy_buffer_start)
{
i = yy_buffer_start;
j = 0;
while (i < yy_buffer_read)
{
yy_buffer[j] = yy_buffer[i];
i++;
j++;
}
yy_buffer_end = yy_buffer_end - yy_buffer_start;
yy_buffer_start = 0;
yy_buffer_read = j;
yy_buffer_index = j;
next_read = yy_reader.Read(yy_buffer, yy_buffer_read,
yy_buffer.Length - yy_buffer_read);
if (next_read <= 0)
{
return((char)YY_EOF);
}
yy_buffer_read = yy_buffer_read + next_read;
}
while (yy_buffer_index >= yy_buffer_read)
{
if (yy_buffer_index >= yy_buffer.Length)
{
yy_buffer = yy_double(yy_buffer);
}
next_read = yy_reader.Read(yy_buffer, yy_buffer_read,
yy_buffer.Length - yy_buffer_read);
if (next_read <= 0)
{
return((char)YY_EOF);
}
yy_buffer_read = yy_buffer_read + next_read;
}
return(yy_buffer[yy_buffer_index++]);
}
private static S3 ParseS3(TextReader reader, int context, int innerContext, IServiceLocator locator)
{
for (int i = 0; i < context; i++)
reader.Read();
var bucket = StringConverter.Parse(reader, innerContext);
var key = StringConverter.Parse(reader, innerContext);
var length = IntConverter.Parse(reader);
var name = StringConverter.Parse(reader, innerContext);
var mimeType = StringConverter.Parse(reader, innerContext);
var metadata = HstoreConverter.Parse(reader, innerContext);
for (int i = 0; i < context; i++)
reader.Read();
reader.Read();
return new S3 { Bucket = bucket, Key = key, Length = length, Name = name, MimeType = mimeType, Metadata = metadata };
}
static string ReadString(System.IO.TextReader f)
{
string s = "";
char c;
for (; ;)
{
c = (char)f.Read();
if (c == ' ')
{
break;
}
if (c == '\n')
{
break;
}
if (c == '\r')
{
continue;
}
s = s + c;
}
// s = s.Trim();
Console.WriteLine(s);
return(s);
}
public bool Parse(TextReader reader)
{
List<EarleyItem> previousState, currentState;
currentState = new List<EarleyItem>();
currentState.Add(new EarleyItem(_start, 0, currentState));
int i;
while((i = reader.Read()) >= 0)
{
char c = (char)i;
previousState = currentState;
currentState = Parse(previousState, c);
}
// finish it off
for (var s = 0; s < currentState.Count; s++)
{
var item = currentState[s];
AddNewItems(currentState, Predict(item, currentState).ToList());
AddNewItems(currentState, Complete(item).ToList());
}
return currentState.Any(x => x.IsComplete && x.Production.Equals(_start));
}
private static object ReadString(System.IO.TextReader reader)
{
const string unicodeDoubleQuotes = @"\u0022";
const string doubleQuotes = @"""";
const string unicodeBackslash = @"\u005c";
const string backslash = @"\";
reader.EatWhiteSpace();
string stringResult = reader.SeekIgnoreEscaped((char)reader.Read());
reader.Read(); // consume escaped character
return(stringResult.Replace(unicodeDoubleQuotes, doubleQuotes)
.Replace(unicodeBackslash, backslash)
.Replace("u005c", backslash)
.Replace("\\\\", "\\"));
}
public static long Parse(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return 0;
return ParseLong(reader, ref cur);
}
public int GetLineCount(TextReader reader)
{
try
{
var buffer = new char[32 * 1024];
ElmentCounter = 1;
int read;
while ((read = reader.Read(buffer, 0, buffer.Length)) > 0)
{
for (int i = 0; i < read; i++)
{
if (buffer[i] == '\n')
ElmentCounter++;
}
}
}
catch (Exception ex)
{
LastError = ex.Message;
return -1;
}
return ElmentCounter;
}
public static float Parse(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return 0;
return ParseFloat(reader, ref cur);
}
public static Guid? ParseNullable(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
return ParseGuid(reader, cur);
}
public static double? ParseNullable(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
return ParseDouble(reader, ref cur);
}
public static Guid Parse(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return Guid.Empty;
return ParseGuid(reader, cur);
}
public static S3 Parse(TextReader reader, int context)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
return ParseS3(reader, context, context << 1, null);
}
public static decimal Parse(TextReader reader)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return 0;
return ParseDecimal(reader, ref cur);
}
private object ReadArray(System.IO.TextReader reader)
{
#pragma warning disable CS8604 // Possible null reference argument.
IList?list = Activator.CreateInstance(DefaultArrayType) as IList;
#pragma warning restore CS8604 // Possible null reference argument.
reader.FastSeek('[');
reader.Read(); // consume the '['
reader.EatWhiteSpace();
bool done = false;
char ch = (char)reader.Peek();
if (ch == ']')
{
done = true;
reader.Read(); // consume the ']'
}
else
{
if (ch != 't' && ch != 'f' && ch != 'n' && Char.IsLetter(ch))
{
throw new InvalidDataException($"LoadArray char {ch} is not allowed after [");
}
}
while (!done)
{
reader.EatWhiteSpace();
list?.Add(Read(reader));
reader.EatWhiteSpace();
ch = (char)reader.Peek();
if (ch == ',')
{
reader.Read(); // consume ','
}
reader.EatWhiteSpace();
ch = (char)reader.Peek();
if (ch == ']')
{
reader.Read(); // consume ']'
done = true;
}
}
#pragma warning disable CS8604 // Possible null reference argument.
return(list.Simplify());
#pragma warning restore CS8604 // Possible null reference argument.
}
public static List<double?> ParseNullableCollection(TextReader reader, int context)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
var espaced = cur != '{';
if (espaced)
{
for (int i = 0; i < context; i++)
reader.Read();
}
var list = new List<double?>();
cur = reader.Peek();
if (cur == '}')
reader.Read();
while (cur != -1 && cur != '}')
{
cur = reader.Read();
if (cur == 'N')
{
cur = reader.Read();
if (cur == 'U')
{
reader.Read();
reader.Read();
list.Add(null);
}
else
{
list.Add(double.NaN);
reader.Read();
}
cur = reader.Read();
}
else
{
list.Add(ParseDouble(reader, ref cur));
}
}
if (espaced)
{
for (int i = 0; i < context; i++)
reader.Read();
}
reader.Read();
return list;
}
private static object ReadBool(System.IO.TextReader reader)
{
reader.EatWhiteSpace();
char ch = (char)reader.Peek();
if (ch == 't')
{
reader.Read(); reader.Read(); reader.Read(); reader.Read(); // read chars t,r,u,e
return(true);
}
reader.Read(); reader.Read(); reader.Read(); reader.Read(); reader.Read(); // read char f,a,l,s,e
return(false);
}
public static void WriteFromStream(
this TextWriter writer, TextReader reader)
{
int c;
while ((c = reader.Read()) >= 0) {
writer.Write((char)c);
}
}
public static void TestNullTextReader(TextReader input)
{
StreamReader sr = input as StreamReader;
if (sr != null)
Assert.True(sr.EndOfStream, "EndOfStream property didn't return true");
input.ReadLine();
input.Dispose();
input.ReadLine();
if (sr != null)
Assert.True(sr.EndOfStream, "EndOfStream property didn't return true");
input.Read();
input.Peek();
input.Read(new char[2], 0, 2);
input.ReadToEnd();
input.Dispose();
}
public static IConvertible Read(TextReader reader)
{
var state = new JavaScriptReader(reader);
var v = state.ReadCore();
state.SkipSpaces();
if(reader.Read() >= 0)
throw state.JsonError(String.Format("extra characters in JSON input"));
return v;
}
private static Thread CopyThread(TextReader rdr, TextWriter wr)
{
var th = new Thread(delegate()
{
for (int i; ((i = rdr.Read()) >= 0); wr.Write((char)i)) ;
});
th.Start();
return th;
}
// reads the content of a request depending on its length
private static string ReadContent(TextReader textReader, int contentLength)
{
var readBuffer = new char[contentLength];
var bytesRead = 0;
while (bytesRead < contentLength)
{
bytesRead += textReader.Read(readBuffer, bytesRead, contentLength - bytesRead);
}
return readBuffer.Aggregate(string.Empty, (current, ch) => current + ch);
}
public static List<bool?> ParseNullableCollection(TextReader reader, int context)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
var espaced = cur != '{';
if (espaced)
{
for (int i = 0; i < context; i++)
reader.Read();
}
var list = new List<bool?>();
cur = reader.Peek();
if (cur == '}')
reader.Read();
while (cur != -1 && cur != '}')
{
cur = reader.Read();
if (cur == 't')
list.Add(true);
else if (cur == 'f')
list.Add(false);
else
{
reader.Read();
reader.Read();
reader.Read();
list.Add(null);
}
cur = reader.Read();
}
if (espaced)
{
for (int i = 0; i < context; i++)
reader.Read();
}
reader.Read();
return list;
}
public static List<DateTime> ParseCollection(TextReader reader, int context)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
var espaced = cur == '"' || cur == '\\';
if (espaced)
{
for (int i = 0; i < context; i++)
cur = reader.Read();
}
var list = new List<DateTime>();
cur = reader.Peek();
if (cur == '}')
reader.Read();
while (cur != -1 && cur != '}')
{
cur = reader.Read();
if (cur == 'N')
{
reader.Read();
reader.Read();
reader.Read();
list.Add(DateTime.MinValue);
}
else
{
list.Add(ParseDate(reader, cur));
}
cur = reader.Read();
}
if (espaced)
{
for (int i = 0; i < context; i++)
reader.Read();
}
reader.Read();
return list;
}
public static List<Guid> ParseCollection(TextReader reader, int context)
{
var cur = reader.Read();
if (cur == ',' || cur == ')')
return null;
var espaced = cur != '{';
if (espaced)
{
for (int i = 0; i < context; i++)
reader.Read();
}
var list = new List<Guid>();
cur = reader.Peek();
if (cur == '}')
reader.Read();
while (cur != -1 && cur != '}')
{
cur = reader.Read();
if (cur == 'N')
{
reader.Read();
reader.Read();
reader.Read();
list.Add(Guid.Empty);
}
else
{
list.Add(ParseCollectionGuid(reader, cur));
}
cur = reader.Read();
}
if (espaced)
{
for (int i = 0; i < context; i++)
reader.Read();
}
reader.Read();
return list;
}
public static int ReadInt32(TextReader reader)
{
int currentChar = 0;
int number = 0;
bool ok = false;
bool minus = reader.Peek() == '-';
if (minus)
{
reader.Read();
}
while (((currentChar = reader.Peek()) > 0) && char.IsNumber((char)currentChar))
{
number = number * 10 + (currentChar - 0x30);
ok = true;
reader.Read();
}
if (!ok)
{
throw new FormatException();
}
return minus ? -number : number;
}
internal static int ReadFully(Reader reader, char[] dest, int offset, int len)
{
int read = 0;
while (read < len)
{
int r = reader.Read(dest, offset + read, len - read);
if (r == 0)
{
break;
}
read += r;
}
return(read);
}
private KeyValuePair<Match, int> Match(TextReader reader, int position)
{
while (reader.Peek() != -1)
{
char c = (char) reader.Read();
++position;
var patternMatch = _machine.Feed(c);
if (patternMatch != null)
{
var match = new Match(position - patternMatch.Length, patternMatch);
return new KeyValuePair<Match, int>(match, position);
}
}
return new KeyValuePair<Match, int>(null, position);
}
public PanGuTokenizer(System.IO.TextReader input)
: base(input)
{
lock (_LockObj)
{
InitPanGuSegment();
}
_InputText = base.input.ReadToEnd();
if (string.IsNullOrEmpty(_InputText))
{
char[] readBuf = new char[1024];
int relCount = base.input.Read(readBuf, 0, readBuf.Length);
StringBuilder inputStr = new StringBuilder(readBuf.Length);
while (relCount > 0)
{
inputStr.Append(readBuf, 0, relCount);
relCount = input.Read(readBuf, 0, readBuf.Length);
}
if (inputStr.Length > 0)
{
_InputText = inputStr.ToString();
}
}
if (string.IsNullOrEmpty(_InputText))
{
_WordList = new WordInfo[0];
}
else
{
global::PanGu.Segment segment = new Segment();
ICollection <WordInfo> wordInfos = segment.DoSegment(_InputText);
_WordList = new WordInfo[wordInfos.Count];
wordInfos.CopyTo(_WordList, 0);
}
}
/// <summary>
/// Determines if the reader is at the end of a line.
/// </summary>
public bool IsEndOfLine()
{
int c;
while (parenLevel > 0)
{
while ((c = text.Peek()) >= 0)
{
if (c == ' ' || c == '\t' || c == '\r' || c == '\n')
{
c = text.Read();
previousChar = c;
continue;
}
if (c == ')')
{
--parenLevel;
c = text.Read();
previousChar = c;
break;
}
return(false);
}
}
if (eolSeen)
{
return(true);
}
while ((c = text.Peek()) >= 0)
{
// Skip space or tab.
if (c == ' ' || c == '\t')
{
c = text.Read();
previousChar = c;
continue;
}
return(c == '\r' || c == '\n' || c == ';');
}
// EOF is end of line
return(true);
}
private void Refill()
{
int newPosition = bufferLength - tokenStart;
if (tokenStart == 0)
{
// token won't fit in buffer
if (buffer is null)
{
// first time: alloc buffer
buffer = new char[2048];
}
else if (bufferLength == buffer.Length)
{
// grow buffer
char[] newBuffer = new char[buffer.Length * 2];
Array.Copy(buffer, 0, newBuffer, 0, bufferLength);
buffer = newBuffer;
}
}
else
{
// shift token to front
Array.Copy(buffer, tokenStart, buffer, 0, newPosition);
}
bufferLength = newPosition; // update state
bufferPosition = newPosition;
bufferStart += tokenStart;
tokenStart = 0;
int charsRead = input.Read(buffer, newPosition, buffer.Length - newPosition);
if (charsRead <= 0)
{
throw new IOException("read past eof");
}
else
{
bufferLength += charsRead;
}
}
/**
* Refills the input buffer.
*
* @return <code>false</code>, iff there was new input.
*
* @exception IOException if any I/O-Error occurs
*/
private bool zzRefill()
{
/* first: make room (if you can) */
if (zzStartRead > 0)
{
Array.Copy(zzBuffer, zzStartRead,
zzBuffer, 0,
zzEndRead - zzStartRead);
/* translate stored positions */
zzEndRead -= zzStartRead;
zzCurrentPos -= zzStartRead;
zzMarkedPos -= zzStartRead;
zzPushbackPos -= zzStartRead;
zzStartRead = 0;
}
/* is the buffer big enough? */
if (zzCurrentPos >= zzBuffer.Length)
{
/* if not: blow it up */
char[] newBuffer = new char[zzCurrentPos * 2];
Array.Copy(zzBuffer, 0, newBuffer, 0, zzBuffer.Length);
zzBuffer = newBuffer;
}
/* finally: fill the buffer with new input */
int numRead = zzReader.Read(zzBuffer, zzEndRead,
zzBuffer.Length - zzEndRead);
if (numRead <= 0)
{
return(true);
}
else
{
zzEndRead += numRead;
return(false);
}
}
/// <summary>Reads a number of characters from the current source TextReader and writes the data to the target array at the specified index.</summary>
/// <param name="sourceTextReader">The source TextReader to read from</param>
/// <param name="target">Contains the array of characteres read from the source TextReader.</param>
/// <param name="start">The starting index of the target array.</param>
/// <param name="count">The maximum number of characters to read from the source TextReader.</param>
/// <returns>The number of characters read. The number will be less than or equal to count depending on the data available in the source TextReader. Returns -1 if the end of the stream is reached.</returns>
public static System.Int32 ReadInput(System.IO.TextReader sourceTextReader, byte[] target, int start, int count)
{
// Returns 0 bytes if not enough space in target
if (target.Length == 0)
{
return(0);
}
char[] charArray = new char[target.Length];
int bytesRead = sourceTextReader.Read(charArray, start, count);
// Returns -1 if EOF
if (bytesRead == 0)
{
return(-1);
}
for (int index = start; index < start + bytesRead; index++)
{
target[index] = (byte)charArray[index];
}
return(bytesRead);
}
public static IEnumerable <string> ReadLines(this System.IO.TextReader reader, char[] delimiter)
{
List <char> chars = new List <char>();
int d = 0; /* Index of the current delimiter char to check against */
while (reader.Peek() >= 0)
{
char c = (char)reader.Read();
if (c == delimiter[d]) /* If the char matches the current delimiter char */
{
d++;
if (d == delimiter.Length) /* If all the delimiter char's were found, add the word and continue */
{
d = 0; /* Reset the delimiter index */
yield return(new String(chars.ToArray()));
chars.Clear();
continue;
}
}
else if (d > 0) /* If the delimiter match failed, retroactively add those char's */
{
for (int i = 0; i < d; i++)
{
chars.Add(delimiter[i]);
}
d = 0;
chars.Add(c);
}
else
{
chars.Add(c);
}
}
}
static bool _innerSkipToNewlineDelimiter(this TextReader reader, string keyword)
{
var i = 0;
var j = 0;
int ci;
var delimiter = Environment.NewLine;
bool secondHit = false;
while ((ci = reader.Read()) != -1)
{
var c = (char)ci;
if (c == delimiter[j])
{
++j;
if (j == delimiter.Length)
{
if (secondHit)
{
break;
}
else
{
while ((ci = reader.Read()) != -1 && ci != delimiter[0] && ((char)ci).IsWhiteSpace())
{
;
}
if (ci == -1)
{
break;
}
else if (ci == delimiter[0])
{
secondHit = true;
j = 1;
continue;
}
else
{
c = (char)ci;
}
}
}
}
else
{
if (c == delimiter[0])
{
j = 1;
}
else
{
j = 0;
}
}
if (c == keyword[i])
{
++i;
if (i == keyword.Length)
{
return(true);
}
}
else
{
if (c == keyword[0])
{
i = 1;
}
else
{
i = 0;
}
}
}
return(false);
}
public override int Read()
{
lock (this){
return(reader.Read());
}
}
public override int Read() => _in.Read();
private void ReadProgramFile(System.IO.TextReader programFile)
{
while (programFile.Peek() != -1)
{
char symbol = (char)programFile.Peek();
if (char.IsWhiteSpace(symbol))
{
// if the symbol is whitespace then move to next symbol
programFile.Read();
}
else if (char.IsLetter(symbol) || symbol == '_')
{
//identify the tokens
StringBuilder token = new StringBuilder();
while (char.IsLetter(symbol) || symbol == '_')
{
token.Append(symbol);
programFile.Read();
if (programFile.Peek() == -1)
{
break;
}
else
{
symbol = (char)programFile.Peek();
}
}
listLexemes.Add(token.ToString());
}
else if (symbol == '"')
{
// string literal
StringBuilder stringLiteral = new StringBuilder();
programFile.Read(); // skip the '"'
if (programFile.Peek() == -1)
{
throw new Exception("String literal is not terminated");
}
while ((symbol = (char)programFile.Peek()) != '"')
{
stringLiteral.Append(symbol);
programFile.Read();
if (programFile.Peek() == -1)
{
throw new Exception("String literal is not terminated");
}
}
// skip the terminating "
programFile.Read();
listLexemes.Add(stringLiteral);
}
else if (char.IsDigit(symbol))
{
// numeric literal
StringBuilder numericLiteral = new StringBuilder();
while (char.IsDigit(symbol))
{
numericLiteral.Append(symbol);
programFile.Read();
if (programFile.Peek() == -1)
{
break;
}
else
{
symbol = (char)programFile.Peek();
}
}
listLexemes.Add(int.Parse(numericLiteral.ToString()));
}
else
{
switch (symbol)
{
case '(':
programFile.Read();
listLexemes.Add(Tokens.OpenParen);
break;
case ')':
programFile.Read();
listLexemes.Add(Tokens.CloseParen);
break;
case '=':
programFile.Read();
listLexemes.Add(Tokens.EqualTo);
break;
case ';':
programFile.Read();
listLexemes.Add(Tokens.Terminator);
break;
case '<':
programFile.Read();
listLexemes.Add(Tokens.LessThan);
break;
case '>':
programFile.Read();
listLexemes.Add(Tokens.GreaterThan);
break;
case ',':
programFile.Read();
listLexemes.Add(Tokens.Comma);
break;
case '/':
programFile.Read();
listLexemes.Add(Tokens.Divide);
break;
case '*':
programFile.Read();
listLexemes.Add(Tokens.Multiply);
break;
case '-':
programFile.Read();
listLexemes.Add(Tokens.Subtract);
break;
case '+':
programFile.Read();
listLexemes.Add(Tokens.Add);
break;
case '{':
programFile.Read();
listLexemes.Add(Tokens.OpenBrac);
break;
case '}':
programFile.Read();
listLexemes.Add(Tokens.CloseBrac);
break;
default:
ExceptionHandler("unidentified symbol '" + symbol + "'");
break;
}
}
}
}
private void Scan(IO.TextReader input)
{
while (input.Peek() != -1)
{
char ch = (char)input.Peek();
// Scan individual tokens
if (char.IsWhiteSpace(ch))
{
// eat the current char and skip ahead!
input.Read();
}
else if (char.IsLetter(ch) || ch == '_')
{
// keyword or identifier
Text.StringBuilder accum = new Text.StringBuilder();
while (char.IsLetter(ch) || ch == '_')
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
break;
}
else
{
ch = (char)input.Peek();
}
}
this.result.Add(accum.ToString());
}
else if (ch == '"')
{
// string literal
Text.StringBuilder accum = new Text.StringBuilder();
input.Read(); // skip the '"'
if (input.Peek() == -1)
{
throw new System.Exception("unterminated string literal");
}
while ((ch = (char)input.Peek()) != '"')// "
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
throw new System.Exception("unterminated string literal");
}
}
// skip the terminating "
input.Read();
this.result.Add(accum);
}
else if (char.IsDigit(ch))
{
// numeric literal
Text.StringBuilder accum = new Text.StringBuilder();
while (char.IsDigit(ch))
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
break;
}
else
{
ch = (char)input.Peek();
}
}
this.result.Add(int.Parse(accum.ToString()));
}
else
{
switch (ch)
{
case '+':
input.Read();
this.result.Add(Scanner.Add);
break;
case '-':
input.Read();
this.result.Add(Scanner.Sub);
break;
case '*':
input.Read();
this.result.Add(Scanner.Mul);
break;
case '/':
input.Read();
this.result.Add(Scanner.Div);
break;
case '=':
input.Read();
this.result.Add(Scanner.Equal);
break;
case ';':
input.Read();
this.result.Add(Scanner.Semi);
break;
default:
throw new System.Exception("Scanner encountered unrecognized character '" + ch + "'");
}
}
}
}
//---------------------------------------------------------------------
public TokenList Tokenize(System.IO.TextReader Text)
{
TokenList tokens = new TokenList();
Token token = new Token();
int nChar = 0;
while (true)
{
int CharCode = Text.Peek();
if (CharCode == -1)
{
break;
}
if (this.WhitespaceChars.Contains("" + (char)CharCode))
{
// Accumulate a Whitespace character.
if (token.Type != TokenType.Whitespace)
{
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Whitespace, "", nChar);
}
token.Text += (char)Text.Read();
nChar++;
}
else if (this.SymbolChars.Contains("" + (char)CharCode))
{
// Accumulate a single Symbol character.
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Symbol, ("" + (char)Text.Read()), nChar);
this.CollectToken_(tokens, token);
nChar++;
token = this.NewToken_(TokenType.Unknown, "", nChar);
}
else if (this.LiteralDelimiters.Contains("" + (char)CharCode))
{
// Accumulate a Literal character.
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Literal, ("" + (char)Text.Read()), nChar);
nChar++;
// Read until the closing delimiter.
while (true)
{
int NextCharCode = Text.Peek();
if (NextCharCode == -1)
{
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Unknown, "", nChar);
break;
}
else if (NextCharCode == CharCode)
{
token.Text += (char)Text.Read();
nChar++;
if (!token.Text.EndsWith(this.LiteralEscapeChar + (char)CharCode))
{
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Unknown, "", nChar);
break;
}
}
else
{
token.Text += (char)Text.Read();
nChar++;
}
}
}
else
{
// Accumulate an identifier.
if (token.Type != TokenType.Identifier)
{
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Identifier, "", nChar);
}
token.Text += (char)Text.Read();
nChar++;
}
}
this.CollectToken_(tokens, token);
// Return the tokens.
return(tokens);
}
public override int Read()
{
return(_in.Read());
}
void Scan(IO.TextReader input)
{
while (input.Peek() != -1)
{
char ch = (char)input.Peek();
if (ch == '%')
{
//eat '%'
input.Read();
Text.StringBuilder accum = new Text.StringBuilder();
ch = (char)input.Peek();
while (ch != '%')
{
accum.Append((char)input.Read());
if (input.Peek() == -1)
{
break;
}
ch = (char)input.Peek();
}
input.Read(); //eat '%'
AppendToken(accum.ToString(), line, 2);
}
else if (ch == ',')
{
input.Read();
}
else if (ch == '$')
{
//omg a function
Text.StringBuilder accum = new Text.StringBuilder();
input.Read();
ch = (char)input.Peek();
while (char.IsLetterOrDigit(ch) || ch == '_')
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
break;
}
ch = (char)input.Peek();
}
if (ch != '(')
{
throw new Exception("Invalid character in function name");
}
AppendToken(accum.ToString(), line, 3);
}
else if (ch == '(')
{
input.Read();
AppendToken("", line, 4);
}
else if (ch == ')')
{
input.Read();
AppendToken("", line, 8);
}
else if (ch == '\r')
{
input.Read(); //eat CR
char c = (char)input.Peek();
if (c == '\n')
{
input.Read(); //eat LF
AppendToken("", line, 7);
}
else
{
line++;
}
}
else if (ch == '#')
{
input.Read();
AppendToken("", line, 9);
}
else
{
Text.StringBuilder accum = new System.Text.StringBuilder();
while (ch != '$' & ch != '%' & ch != '\r' & ch != ')')
{
if (ch == '\\')
{
//skip escape char and insert next one immediately
input.Read();
}
accum.Append((char)input.Read());
if (input.Peek() == -1)
{
break;
}
else if (input.Peek() == ',')
{
input.Read();
break;
}
ch = (char)input.Peek();
}
AppendToken(accum.ToString(), line, 1);
}
}
}
private object ReadObject(System.IO.TextReader reader)
{
const char objectOpenCharacter = '{';
const char objectCloseCharacter = '}';
const char comma = ',';
#pragma warning disable CS8600 // Converting null literal or possible null value to non-nullable type.
IDictionary dictionary = null;
#pragma warning restore CS8600 // Converting null literal or possible null value to non-nullable type.
if (ObjectCount == 0)
{
if (DefaultDocumentType == null)
{
throw new InvalidOperationException("DefaultDocumentType is null");
}
#pragma warning disable CS8600 // Converting null literal or possible null value to non-nullable type.
dictionary = Activator.CreateInstance(DefaultDocumentType) as IDictionary;
#pragma warning restore CS8600 // Converting null literal or possible null value to non-nullable type.
if (dictionary == null)
{
throw new InvalidOperationException($"Unable to create instance of {DefaultDocumentType.FullName}");
}
}
else
{
if (CreateDefaultObject != null)
{
dictionary = CreateDefaultObject();
if (dictionary == null)
{
throw new InvalidOperationException("CreateDefaultObject returned null");
}
}
else
{
if (DefaultObjectType == null)
{
throw new InvalidOperationException("DefaultObjectType is null");
}
#pragma warning disable CS8600 // Converting null literal or possible null value to non-nullable type.
dictionary = Activator.CreateInstance(DefaultObjectType) as IDictionary;
#pragma warning restore CS8600 // Converting null literal or possible null value to non-nullable type.
if (dictionary == null)
{
throw new InvalidOperationException($"Unable to create isntance of {DefaultObjectType.FullName}");
}
}
}
ObjectCount++;
reader.FastSeek(objectOpenCharacter);
reader.Read(); // consume the '{'
reader.EatWhiteSpace();
bool done = false;
if ((char)(reader.Peek()) == objectCloseCharacter)// '}')
{
done = true;
reader.Read(); // consume the '}'
}
while (!done)
{
reader.EatWhiteSpace();
string?key = ReadString(reader) as string;
/*
#if DEBUG
* if (key == "string_symbol")
* {
* int x = 0;
* }
#endif*/
reader.EatWhiteSpace();
reader.Read(); //consume ':'
#pragma warning disable CS8604 // Possible null reference argument.
dictionary[key] = Read(reader);
#pragma warning restore CS8604 // Possible null reference argument.
reader.EatWhiteSpace();
char ch = (char)reader.Peek();
if (ch == comma)
{
reader.Read(); // consume ','
}
reader.EatWhiteSpace();
ch = (char)reader.Peek();
if (ch == objectCloseCharacter)//'}')
{
reader.Read();
done = true;
}
}
string?type = dictionary.Get <string>("Type", "");
if (type.Length > 0 && ConversionTypeNames.ContainsKey(type) && !ConversionTypeNames[type].IsInstanceOfType(dictionary))
{
if (!(Activator.CreateInstance(ConversionTypeNames[type]) is IDictionary converted))
{
throw new InvalidOperationException($"Unable to create instance of {ConversionTypeNames[type].FullName}");
}
foreach (object?key in dictionary.Keys)
{
#pragma warning disable CS8604 // Possible null reference argument.
if (!converted.Contains(key))
#pragma warning restore CS8604 // Possible null reference argument.
{
#pragma warning disable CS8604 // Possible null reference argument.
#pragma warning disable CS8604 // Possible null reference argument.
converted.Add(key, dictionary[key]);
#pragma warning restore CS8604 // Possible null reference argument.
#pragma warning restore CS8604 // Possible null reference argument.
}
}
return(converted);
}
return(dictionary);
}
private void Scan(IO.TextReader input)
{
while (input.Peek() != -1)
{
char caracter = (char)input.Peek();
// Scan individual tokens
if (char.IsWhiteSpace(caracter))
{
// eat the current char and skip ahead!
input.Read();
}
else if (char.IsLetter(caracter) || caracter == '_')
{
// keyword or identifier
Text.StringBuilder accum = new Text.StringBuilder();
while (char.IsLetter(caracter) || caracter == '_')
{
accum.Append(caracter);
input.Read();
if (input.Peek() == -1)
{
break;
}
else
{
caracter = (char)input.Peek();
}
}
this.resultado.Add(accum.ToString());
}
else if (caracter == '"')
{
// string literal
Text.StringBuilder accum = new Text.StringBuilder();
input.Read(); // skip the '"'
if (input.Peek() == -1)
{
throw new System.Exception("Cadena sin terminar");
}
while ((caracter = (char)input.Peek()) != '"')
{
accum.Append(caracter);
input.Read();
if (input.Peek() == -1)
{
throw new System.Exception("Cadena sin terminar");
}
}
// skip the terminating "
input.Read();
this.resultado.Add(accum);
}
else if (char.IsDigit(caracter))
{
// numeric literal
Text.StringBuilder accum = new Text.StringBuilder();
while (char.IsDigit(caracter))
{
accum.Append(caracter);
input.Read();
if (input.Peek() == -1)
{
break;
}
else
{
caracter = (char)input.Peek();
}
}
this.resultado.Add(int.Parse(accum.ToString()));
}
else
{
switch (caracter)
{
case '+':
input.Read();
this.resultado.Add(Scanner.Sum);
break;
case '-':
input.Read();
this.resultado.Add(Scanner.Res);
break;
case '*':
input.Read();
this.resultado.Add(Scanner.Mul);
break;
case '/':
input.Read();
this.resultado.Add(Scanner.Div);
break;
case '=':
input.Read();
if (input.Peek() == '=')
{
input.Read();
this.resultado.Add(Scanner.Eq);
}
else
{
this.resultado.Add(Scanner.Igual);
}
break;
case ';':
input.Read();
this.resultado.Add(Scanner.PyC);
break;
case '>':
input.Read();
if (input.Peek() == '=')
{
input.Read();
this.resultado.Add(Scanner.Gte);
}
else
{
this.resultado.Add(Scanner.Gt);
}
break;
case '<':
input.Read();
if (input.Peek() == '=')
{
input.Read();
this.resultado.Add(Scanner.Lte);
}
else
{
this.resultado.Add(Scanner.Lt);
}
break;
case '!':
input.Read();
if (input.Peek() == '=')
{
input.Read();
this.resultado.Add(Scanner.Neq);
}
else
{
throw new System.Exception("No se reconoce el siguiente caracter: '" + caracter + "'");
}
break;
default:
throw new System.Exception("No se reconoce el siguiente caracter: '" + caracter + "'");
}
}
}
}
private void Scan(IO.TextReader input)
{
while (input.Peek() != -1)
{
char ch = (char)input.Peek();
// Scan individual tokens
if (ch != '\r' && char.IsWhiteSpace(ch) || ch == ',')
{
// eat the current char and skip ahead!
input.Read();
}
else if (char.IsLetter(ch) || ch == '_')
{
// keyword or identifier
Text.StringBuilder accum = new Text.StringBuilder();
while (char.IsLetter(ch) || ch == '_')
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
break;
}
else
{
ch = (char)input.Peek();
}
}
this.result.Add(accum.ToString());
}
else if (ch == '"')
{
// string literal
Text.StringBuilder accum = new Text.StringBuilder();
input.Read(); // skip the '"'
if (input.Peek() == -1)
{
throw new System.Exception("unterminated string literal");
}
while ((ch = (char)input.Peek()) != '"')
{
byte[] unicode = Text.Encoding.Default.GetBytes(ch.ToString(CultureInfo.CurrentCulture));
byte[] ascii = Text.Encoding.Convert(Text.Encoding.Unicode, Text.Encoding.ASCII, unicode);
char[] asciiChars = new char[Text.Encoding.ASCII.GetCharCount(ascii, 0, ascii.Length)];
Text.Encoding.ASCII.GetChars(ascii, 0, ascii.Length, asciiChars, 0);
//accum.Append(asciiChars);
accum.Append(ch);
//accum.Append(ch.ToString(CultureInfo.CurrentCulture));
input.Read();
if (input.Peek() == -1)
{
throw new System.Exception("unterminated string literal");
}
}
// skip the terminating "
input.Read();
this.result.Add(accum);
}
else if (char.IsDigit(ch))
{
bool isDouble = false;
// numeric literal
Text.StringBuilder accum = new Text.StringBuilder();
while (char.IsDigit(ch))
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
break;
}
else
{
ch = (char)input.Peek();
}
}
if (ch == '.')
{
accum.Append(ch);
input.Read();
ch = (char)input.Peek();
while (char.IsDigit(ch))
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
break;
}
else
{
ch = (char)input.Peek();
}
}
isDouble = true;
}
if (isDouble)
{
this.result.Add(double.Parse(accum.ToString().Replace(".", ",")));
}
else
{
this.result.Add(int.Parse(accum.ToString()));
}
}
else
{
switch (ch)
{
case '+':
input.Read();
this.result.Add(Scanner.Add);
break;
case '-':
input.Read();
this.result.Add(Scanner.Sub);
break;
case '*':
input.Read();
this.result.Add(Scanner.Mul);
break;
case '/':
input.Read();
this.result.Add(Scanner.Div);
break;
case '=':
input.Read();
this.result.Add(Scanner.Equal);
break;
case '>': // > | >=
input.Read();
ch = (char)input.Peek();
if (ch == '=')
{
this.result.Add(Scanner.MoreEqual);
input.Read();
}
else
{
this.result.Add(Scanner.More);
}
break;
case '<': // < | <=
input.Read();
ch = (char)input.Peek();
if (ch == '=')
{
this.result.Add(Scanner.LessEqual);
input.Read();
}
else
{
this.result.Add(Scanner.Less);
}
break;
case '\r':
input.Read();
if (!this.result[this.result.Count - 1].ToString().Equals("then") && !this.result[this.result.Count - 1].ToString().Equals("else") &&
!this.result[this.result.Count - 1].ToString().Equals("to") && !this.result[this.result.Count - 1].ToString().Equals("do") &&
this.result[this.result.Count - 1] != Scanner.Semi)
{
this.result.Add(Scanner.Semi);
}
break;
default:
throw new System.Exception("Scanner encountered unrecognized character '" + ch + "'");
}
}
}
}
private void Scan(IO.TextReader input)
{
while (input.Peek() != -1)
{
char ch = (char)input.Peek();
// Scan individual tokens
if (char.IsWhiteSpace(ch))
{
// eat the current char and skip ahead!
input.Read();
}
else if (char.IsLetter(ch) || ch == '_')
{
// keyword or identifier
Text.StringBuilder accum = new Text.StringBuilder();
while (char.IsLetter(ch) || ch == '_' || ch == '.')
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
break;
}
else
{
ch = (char)input.Peek();
}
}
if (IsAmExpression(accum) || IsFunctionCall(accum) || IsNewObjCall(accum) || IsLogicalExpression(accum))
{
;
}
else
{
if (result.Count > 0 && this.result[result.Count - 1] == Scanner.Call)
{
this.result.Add(languageSetting[accum.ToString()] ?? accum.ToString());
}
else
{
this.result.Add(accum.ToString());
}
}
}
else if (ch == '"')
{
// string literal
Text.StringBuilder accum = new Text.StringBuilder();
input.Read(); // skip the '"'
if (input.Peek() == -1)
{
throw new System.Exception("unterminated string literal");
}
while ((ch = (char)input.Peek()) != '"')
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
throw new System.Exception("unterminated string literal");
}
}
// skip the terminating"
input.Read();
this.result.Add(accum);
}
else if (char.IsDigit(ch))
{
// numeric literal
Text.StringBuilder accum = new Text.StringBuilder();
while (char.IsDigit(ch))
{
accum.Append(ch);
input.Read();
if (input.Peek() == -1)
{
break;
}
else
{
ch = (char)input.Peek();
}
}
this.result.Add(int.Parse(accum.ToString()));
}
//Obselete Code
else
{
switch (ch)
{
case '+':
input.Read();
this.result.Add(Scanner.Add);
break;
case '-':
input.Read();
this.result.Add(Scanner.Sub);
break;
case '*':
input.Read();
this.result.Add(Scanner.Mul);
break;
case '/':
input.Read();
this.result.Add(Scanner.Div);
break;
case '=':
input.Read();
this.result.Add(Scanner.Equal);
break;
case ';':
input.Read();
this.result.Add(Scanner.Semi);
break;
case '(':
input.Read();
this.result.Add(Scanner.OpeningParentheses);
break;
case ')':
input.Read();
this.result.Add(Scanner.ClosingParentheses);
break;
case ',':
input.Read();
this.result.Add(Scanner.Comma);
break;
default:
throw new System.Exception("Scanner encountered unrecognized character '" + ch + "'");
}
}
}
}
static string _innerReadToNewlineDelimiter(this TextReader reader, string keyword, bool returnIfKeywordNotFound)
{
var i = 0;
var j = 0;
int ci;
var sb = StringBuilderCache.Acquire();
var delimiter = Environment.NewLine;
bool secondHit = false;
while ((ci = reader.Read()) != -1)
{
var c = (char)ci;
if (c == delimiter[j])
{
++j;
if (j == delimiter.Length)
{
if (secondHit)
{
break;
}
else
{
while ((ci = reader.Read()) != -1 && ci != delimiter[0] && ((char)ci).IsWhiteSpace())
{
;
}
if (ci == -1)
{
break;
}
else if (ci == delimiter[0])
{
secondHit = true;
j = 1;
continue;
}
else
{
c = (char)ci;
}
}
}
}
else if (c == delimiter.Last())
{
while ((ci = reader.Read()) != -1 && ci != delimiter[0] && ((char)ci).IsWhiteSpace())
{
;
}
if (ci == -1)
{
break;
}
else if (ci == delimiter[0])
{
secondHit = true;
j = 1;
continue;
}
else
{
c = (char)ci;
}
}
else
{
if (c == delimiter[0])
{
j = 1;
}
else
{
j = 0;
}
}
if (c == keyword[i])
{
++i;
if (i == keyword.Length)
{
return(StringBuilderCache.GetStringAndRelease(sb));
}
}
else
{
if (i != 0)
{
sb.Append(keyword.Substring(0, i));
i = 0;
}
if (c == keyword[0])
{
++i;
}
else
{
sb.Append(c);
}
}
}
if (returnIfKeywordNotFound)
{
return(StringBuilderCache.GetStringAndRelease(sb));
}
else
{
StringBuilderCache.Release(sb);
return(null);
}
}
/// <summary>
/// Enumerates non whitespace tokens by using the given test reader.
/// </summary>
/// <remarks>This tokenizer does not return line number and column information.</remarks>
/// <param name="inputReader">Input reader containing textual data.</param>
/// <returns>Enumeration of non-whitespace tokens.</returns>
public IEnumerable <IToken> EnumerateTokens(System.IO.TextReader inputReader)
{
if (inputReader == null)
{
yield break;
}
// Initialize parser
ParserState currentState = ParserState.Init;
int tokenStart = -1;
int position = 0;
StringBuilder tokenBuilder = new StringBuilder(1024);
// Read all characters, one by one
for (int c = inputReader.Read(); c >= 0; c = inputReader.Read())
{
char currentChar = (char)c;
switch (currentState)
{
case ParserState.Init:
{
if (char.IsWhiteSpace(currentChar) || currentChar == '\n' || currentChar == '\r')
{
currentState = ParserState.WhiteSpace;
}
else
{
currentState = ParserState.NonWhiteSpace;
tokenBuilder.Append(currentChar);
tokenStart = position;
}
}
break;
case ParserState.WhiteSpace:
{
if (!(char.IsWhiteSpace(currentChar) || currentChar == '\n' || currentChar == '\r'))
{
currentState = ParserState.NonWhiteSpace;
tokenBuilder.Clear();
tokenBuilder.EnsureCapacity(1024);
tokenBuilder.Append(currentChar);
tokenStart = position;
}
}
break;
case ParserState.NonWhiteSpace:
{
if ((char.IsWhiteSpace(currentChar) || currentChar == '\n' || currentChar == '\r'))
{
yield return(new Token(tokenBuilder.ToString(), TOKEN_TYPE_NONWHITESPACE, tokenStart, position - tokenStart));
currentState = ParserState.WhiteSpace;
tokenBuilder.Clear();
tokenBuilder.EnsureCapacity(1024);
tokenStart = -1;
}
else
{
tokenBuilder.Append(currentChar);
}
}
break;
}
++position;
}
if (tokenStart >= 0 && tokenBuilder.Length > 0)
{
yield return(new Token(tokenBuilder.ToString(), "NonWhitespaceToken", tokenStart, position - tokenStart));
}
}
public override int yylex()
{
char ch;
char peek;
int ord = reader.Read();
//this.text.Clear();
this.text.Length = 0;
//
// Must check for EOF
//
if (ord == -1)
{
return((int)Tokens.EOF);
}
else
{
ch = (char)ord;
}
if (ch == '\n')
{
linenum++;
return(yylex());
}
else if (char.IsWhiteSpace(ch))
{ // Skip white space
while (char.IsWhiteSpace(peek = (char)reader.Peek()))
{
ord = reader.Read();
if (ord == (int)'\n')
{
linenum++;
continue;
}
}
return(yylex());
}
else if (char.IsDigit(ch))
{
text.Append(ch);
while (char.IsDigit(peek = (char)reader.Peek()))
{
text.Append((char)reader.Read());
}
if ((peek = (char)reader.Peek()) == '.')
{
text.Append((char)reader.Read());
}
while (char.IsDigit(peek = (char)reader.Peek()))
{
text.Append((char)reader.Read());
}
try
{
yylval = Parser.MakeNumLeaf(double.Parse(text.ToString()));
return((int)Tokens.NUMBER);
}
catch (FormatException)
{
this.yyerror("Illegal number \"{0}\"", text);
return((int)Tokens.error);
}
}
else if (char.IsLetter(ch))
{
text.Append(char.ToLower(ch));
while (char.IsLetter(peek = (char)reader.Peek()))
{
text.Append(char.ToLower((char)reader.Read()));
}
switch (text.ToString())
{
case "if":
return((int)Tokens.IF);
case "fi":
return((int)Tokens.END_IF);
case "do":
return((int)Tokens.DO);
case "od":
return((int)Tokens.END_DO);
case "print":
return((int)Tokens.PRINT);
default:
if (text.Length >= 1)
{
yylval = Parser.MakeIdLeaf(text.ToString());
return((int)Tokens.ID);
}
else
{
this.yyerror("Illegal name \"{0}\"", text);
return((int)Tokens.error);
}
}
}
else
{
switch (ch)
{
case '.':
case '+':
case '-':
case '*':
case '/':
case '(':
case ')':
case '%':
case '=':
case ':':
case '>':
case '<':
case '!':
case ',':
case ';':
case '|':
return(ch);
default:
yyerror("Illegal character '{0}' on line '{1}'", ch, linenum);
return(yylex());
}
}
}