public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = text.Length;
var c = text[context.Index];
if (c == '\'')
{
var initialIndex = context.Index;
var initialLine = context.Line;
var index = initialIndex + 1; // skip '''
int delta;
while (true)
{
if (index == length)
{
delta = index - initialIndex;
var column = context.Column + delta;
throw LexingHelper.CreateUnclosedStringException(new Position(initialLine, column));
}
c = text[index];
if (LexingHelper.IsCaretControl(c))
{
delta = index - initialIndex;
var column = context.Column + delta;
throw LexingHelper.CreateNewLineInStringException(new Position(initialLine, column));
}
index++;
if (c == '\'')
{
break;
}
}
delta = index - initialIndex;
var str = text.Substring(initialIndex + 1, delta - 2);
var token = new TextToken(
StringTextClass.Instance,
SingleQuoteTextDecoration.Instance,
str,
new Position(context.Line, context.Column),
delta);
context.Advance(delta, 0, context.Column + delta);
return(token);
}
return(null);
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = text.Length;
var c = text[context.Index];
if (LexingHelper.IsLatinLetter(c) || c == '_')
{
var initialIndex = context.Index;
var index = initialIndex + 1;
var column = context.Column + 1;
while (true)
{
if (index == length)
{
break;
}
c = text[index];
if (
LexingHelper.IsInlineWhiteSpaceOrCaretControl(c) ||
LexingHelper.IsStandardPunctuationChar(c))
{
break;
}
if (c == '_' || LexingHelper.IsLatinLetter(c) || LexingHelper.IsDigit(c))
{
index++;
column++;
continue;
}
return(null);
}
var delta = index - initialIndex;
var str = text.Substring(initialIndex, delta);
context.Advance(delta, 0, column);
return(new TextToken(
WordTextClass.Instance,
NoneTextDecoration.Instance,
str,
new Position(context.Line, column), delta));
}
return(null);
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = text.Length;
var c = text[context.Index];
if (IsPathFirstChar(c))
{
var initialIndex = context.Index;
var index = initialIndex + 1;
while (true)
{
if (index == length)
{
break;
}
c = text[index];
if (IsPathInnerChar(c))
{
index++;
continue;
}
if (LexingHelper.IsInlineWhiteSpaceOrCaretControl(c))
{
break;
}
return(null);
}
var delta = index - initialIndex;
var str = text.Substring(initialIndex, delta);
var position = new Position(context.Line, context.Column);
context.Advance(delta, 0, context.Column + delta);
var token = new TextToken(
PathTextClass.Instance,
NoneTextDecoration.Instance,
str,
position,
delta);
return(token);
}
return(null);
}
public PunctuationToken(
char c,
Position position,
int consumedLength)
: base(position, consumedLength)
{
if (!LexingHelper.IsStandardPunctuationChar(c))
{
throw new ArgumentOutOfRangeException(nameof(c));
}
this.Value = c;
}
private bool IsAcceptableIntegerTerminator(char c)
{
if (LexingHelper.IsInlineWhiteSpaceOrCaretControl(c))
{
return(true);
}
if (c.IsIn('(', ')', ','))
{
return(true);
}
return(false);
}
private static bool IntegerTerminatorPredicate(char c)
{
if (LexingHelper.IsInlineWhiteSpaceOrCaretControl(c))
{
return(true);
}
if (TinyLispHelper.IsPunctuation(c))
{
return(true);
}
return(false);
}
public ExactPunctuationNode(
char c,
Action <ActionNode, IToken, IResultAccumulator> action,
INodeFamily family,
string name)
: base(action, family, name)
{
if (!LexingHelper.IsStandardPunctuationChar(c))
{
throw new ArgumentOutOfRangeException(nameof(c));
}
this.Value = c;
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = context.Length;
var c = text[context.Index];
if (c == ';')
{
var initialIndex = context.Index;
var index = initialIndex + 1; // skip ';'
var column = context.Column + 1; // skip ';'
while (true)
{
if (index == length)
{
context.Advance(index - initialIndex, 0, column);
return(null);
}
c = text[index];
if (LexingHelper.IsCaretControl(c))
{
context.Advance(index - initialIndex, 0, column);
return(null);
}
index++;
column++;
}
}
else
{
return(null);
}
}
public IToken Produce()
{
var context = this.Context;
var initialIndex = context.Index;
var text = context.Text;
var length = context.Length;
var c = text[initialIndex];
if (!LexingHelper.IsInlineWhiteSpaceOrCaretControl(c))
{
return(null);
}
var currentIndex = initialIndex;
var initialLine = context.Line;
var initialColumn = context.Column;
var lineShift = 0;
var column = context.Column;
while (true)
{
if (currentIndex == length)
{
context.Advance(currentIndex - initialIndex, lineShift, column);
return(null);
}
c = text[currentIndex];
switch (c)
{
case '\t':
case ' ':
currentIndex++;
column++;
break;
case LexingHelper.CR:
currentIndex++;
lineShift++;
column = 0;
if (currentIndex < length)
{
var nextChar = text[currentIndex];
if (nextChar == LexingHelper.LF)
{
currentIndex++;
}
}
break;
case LexingHelper.LF:
currentIndex++;
lineShift++;
column = 0;
break;
default:
var delta = currentIndex - initialIndex;
var token = this.ProduceImpl(new Position(initialLine, initialColumn), delta);
if (currentIndex > initialIndex)
{
context.Advance(delta, lineShift, column);
}
return(token);
}
}
}
private bool IsAcceptableIntegerTerminator(char c)
{
return(LexingHelper.IsInlineWhiteSpaceOrCaretControl(c));
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = context.Length;
var c = text[context.Index];
if (LexingHelper.IsIntegerFirstChar(c))
{
var initialIndex = context.Index;
var initialColumn = context.Column;
var digitsCount = 1;
if (c == '+' || c == '-')
{
digitsCount = 0;
}
var gotPlusSign = c == '+';
var index = initialIndex + 1;
var column = context.Column + 1;
while (true)
{
if (index == length)
{
break;
}
c = text[index];
if (LexingHelper.IsDigit(c))
{
index++;
column++;
digitsCount++;
}
else
{
if (_acceptableTerminatorPredicate(c))
{
break;
}
else
{
return(null);
}
}
}
if (digitsCount == 0)
{
return(null);
}
var delta = index - initialIndex;
var defect = gotPlusSign ? 1 : 0;
var intString = text.Substring(initialIndex + defect, delta - defect);
if (int.TryParse(intString, out var dummy))
{
context.Advance(delta, 0, column);
return(new IntegerToken(intString, new Position(context.Line, initialColumn), delta));
}
else
{
return(null);
}
}
else
{
return(null);
}
}
public static bool IsPathFirstChar(char c) =>
LexingHelper.IsDigit(c) ||
LexingHelper.IsLatinLetter(c) ||
c.IsIn('\\', '/', '.', '!', '~', '$', '%', ';', '_');
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = text.Length;
var c = text[context.Index];
if (c == '-')
{
var initialIndex = context.Index;
var index = initialIndex + 1;
int delta;
var gotGoodChars = false;
while (true)
{
if (index == length)
{
if (text[index - 1] == '-')
{
return(null);
}
break;
}
c = text[index];
delta = index - initialIndex;
if (c == '-')
{
if (delta == 1)
{
index++;
continue;
}
if (delta == 2)
{
if (text[index - 1] == '-')
{
return(null); // "---" at beginning
}
index++;
continue;
}
if (text[index - 1] == '-')
{
return(null); // "--" inside key
}
index++;
continue;
}
if (LexingHelper.IsDigit(c) ||
LexingHelper.IsLatinLetter(c))
{
gotGoodChars = true;
index++;
continue;
}
if (LexingHelper.IsInlineWhiteSpaceOrCaretControl(c))
{
if (text[index - 1] == '-')
{
return(null);
}
break;
}
return(null);
}
if (!gotGoodChars)
{
return(null);
}
delta = index - initialIndex;
var str = text.Substring(initialIndex, delta);
var position = new Position(context.Line, context.Column);
context.Advance(delta, 0, context.Column + delta);
var token = new TextToken(
KeyTextClass.Instance,
NoneTextDecoration.Instance,
str,
position,
delta);
return(token);
}
return(null);
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = context.Length;
var c = text[context.Index];
if (c == '"')
{
var initialIndex = context.Index;
var initialLine = context.Line;
var index = initialIndex + 1; // skip '"'
var lineShift = 0;
var column = context.Column + 1; // skip '"'
while (true)
{
if (index == length)
{
throw LexingHelper.CreateUnclosedStringException(new Position(initialLine + lineShift, column));
}
c = text[index];
switch (c)
{
case LexingHelper.CR:
index++;
lineShift++;
column = 0;
if (index < length)
{
var nextChar = text[index];
if (nextChar == LexingHelper.LF)
{
index++;
}
}
else
{
throw LexingHelper.CreateUnclosedStringException(new Position(
context.Line + lineShift,
column));
}
continue;
case LexingHelper.LF:
index++;
lineShift++;
column = 0;
continue;
}
index++;
column++;
if (c == '"')
{
break;
}
}
var delta = index - initialIndex;
var str = text.Substring(initialIndex + 1, delta - 2);
var token = new TextToken(
StringTextClass.Instance,
DoubleQuoteTextDecoration.Instance,
str,
new Position(context.Line, context.Column),
delta);
context.Advance(delta, lineShift, column);
return(token);
}
return(null);
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = text.Length;
var c = text[context.Index];
if (OpeningDelimiters.Contains(c) ||
c == '_' ||
LexingHelper.IsLatinLetter(c))
{
char?openingDelimiter = OpeningDelimiters.Contains(c) ? c : (char?)null;
var initialIndex = context.Index;
var index = initialIndex + 1;
while (true)
{
if (index == length)
{
if (openingDelimiter.HasValue)
{
var delta = index - initialIndex;
var column = context.Column + delta;
this.ThrowUnclosedIdentifierException(context.Line, column);
}
break;
}
c = text[index];
if (c == '_' || LexingHelper.IsLatinLetter(c) || LexingHelper.IsDigit(c))
{
index++;
continue;
}
if (ClosingDelimiters.Contains(c))
{
if (index - initialIndex > 2)
{
if (openingDelimiter.HasValue)
{
if (openingDelimiter.Value == ReverseDelimiters[c])
{
index++;
var delta = index - initialIndex;
var column = context.Column + delta;
var str = text.Substring(initialIndex + 1, delta - 2);
var position = new Position(context.Line, context.Column);
context.Advance(delta, 0, column);
return(new TextToken(
SqlIdentifierTextClass.Instance,
NoneTextDecoration.Instance,
str,
position,
delta));
}
else
{
var delta = index - initialIndex;
var column = context.Column + delta;
this.ThrowUnclosedIdentifierException(context.Line, column);
}
}
else
{
var delta = index - initialIndex;
var column = context.Column + delta;
throw new LexingException($"Unexpected delimiter: '{c}'.", new Position(context.Line, column));
}
}
else
{
var delta = index - initialIndex;
var column = context.Column + delta;
throw new LexingException($"Unexpected delimiter: '{c}'.", new Position(context.Line, column));
}
}
}
}
return(null);
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = context.Length;
var c = text[context.Index];
if (TinyLispHelper.IsAcceptableSymbolNameChar(c))
{
var gotSign = c == '+' || c == '-';
var pureDigits = 0;
if (!gotSign)
{
pureDigits = LexingHelper.IsDigit(c) ? 1 : 0;
}
var gotNonDigits = false;
var initialIndex = context.Index;
var initialColumn = context.Column;
var index = initialIndex + 1;
var column = context.Column + 1;
while (true)
{
if (index == length)
{
break;
}
c = text[index];
if (c == ':')
{
throw new LexingException("Bad symbol name.", new Position(context.Line, context.Column));
}
if (!TinyLispHelper.IsAcceptableSymbolNameChar(c))
{
break;
}
if (LexingHelper.IsDigit(c))
{
if (!gotNonDigits)
{
pureDigits++;
}
}
else
{
gotNonDigits = true;
pureDigits = 0;
}
index++;
column++;
}
var couldBeInt = pureDigits > 0;
if (couldBeInt)
{
throw new LexingException("Symbol producer delivered an integer.", new Position(context.Line, context.Column));
}
var delta = index - initialIndex;
var str = text.Substring(initialIndex, delta); // todo: use ReadOnlySpan<char> instead of Substring everywhere.
var symbolToken = new LispSymbolToken(str, new Position(context.Line, initialColumn), delta);
context.Advance(delta, 0, column);
return(symbolToken);
}
else
{
return(null);
}
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = context.Length;
var c = text[context.Index];
if (c == '"')
{
var initialIndex = context.Index;
var initialLine = context.Line;
var index = initialIndex + 1; // skip '"'
int delta;
var sb = new StringBuilder();
while (true)
{
if (index == length)
{
var column = context.Column + (index - initialIndex);
throw LexingHelper.CreateUnclosedStringException(new Position(
initialLine,
column));
}
c = text[index];
if (LexingHelper.IsCaretControl(c))
{
var column = context.Column + (index - initialIndex);
throw LexingHelper.CreateNewLineInStringException(new Position(initialLine, column));
}
if (c == '\\')
{
if (index + 1 == length)
{
throw LexingHelper.CreateUnclosedStringException(new Position(
initialLine,
length));
}
var nextChar = text[index + 1];
if (nextChar == 'u')
{
var remaining = length - (index + 1);
if (remaining < 5)
{
delta = index - initialIndex;
var column = context.Column + delta;
this.ThrowBadEscapeException(initialLine, column);
}
var hexNumString = text.Substring(index + 2, 4);
var codeParsed = int.TryParse(
hexNumString,
NumberStyles.HexNumber,
CultureInfo.InvariantCulture,
out var code);
if (!codeParsed)
{
delta = index - initialIndex;
var column = context.Column + delta;
this.ThrowBadEscapeException(initialLine, column);
}
var unescapedChar = (char)code;
sb.Append(unescapedChar);
index += 6;
continue;
}
else
{
var replacement = GetReplacement(nextChar);
if (replacement.HasValue)
{
sb.Append(replacement);
index += 2;
continue;
}
else
{
delta = index - initialIndex;
var column = context.Column + delta;
this.ThrowBadEscapeException(initialLine, column);
}
}
}
index++;
if (c == '"')
{
break;
}
sb.Append(c);
}
delta = index - initialIndex;
var str = sb.ToString();
var token = new TextToken(
StringTextClass.Instance,
DoubleQuoteTextDecoration.Instance,
str,
new Position(context.Line, context.Column),
delta);
context.Advance(delta, 0, context.Column + delta);
return(token);
}
return(null);
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = text.Length;
string start = null;
foreach (var possibleStart in PossibleStarts)
{
if (context.StartsWith(possibleStart))
{
start = possibleStart;
break;
}
}
if (start == null)
{
return(null);
}
var initialIndex = context.Index;
var startIndex = initialIndex + start.Length;
var index = startIndex;
while (true)
{
if (index == length)
{
break;
}
var c = text[index];
if (LexingHelper.IsInlineWhiteSpaceOrCaretControl(c))
{
break;
}
index++;
}
if (index == startIndex)
{
return(null);
}
var delta = index - initialIndex;
var str = text.Substring(initialIndex, delta);
var position = new Position(context.Line, context.Column);
context.Advance(delta, 0, context.Column + delta);
var token = new TextToken(
UrlTextClass.Instance,
NoneTextDecoration.Instance,
str,
position,
delta);
return(token);
}
public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = text.Length;
var c = text[context.Index];
if (LexingHelper.IsDigit(c) ||
LexingHelper.IsLatinLetter(c))
{
var initialIndex = context.Index;
var index = initialIndex + 1;
while (true)
{
if (index == length)
{
if (text[index - 1] == '-')
{
return(null);
}
break;
}
c = text[index];
if (c == '-')
{
if (text[index - 1] == '-')
{
return(null); // two '-' in a row
}
index++;
continue;
}
if (c == '=' || LexingHelper.IsInlineWhiteSpaceOrCaretControl(c))
{
if (text[index - 1] == '-')
{
return(null);
}
break;
}
var valid =
LexingHelper.IsDigit(c) ||
LexingHelper.IsLatinLetter(c);
if (!valid)
{
return(null);
}
index++;
}
var delta = index - initialIndex;
var str = text.Substring(initialIndex, delta);
var position = new Position(context.Line, context.Column);
context.Advance(delta, 0, context.Column + delta);
var token = new TextToken(
TermTextClass.Instance,
NoneTextDecoration.Instance,
str,
position,
delta);
return(token);
}
return(null);
}
