public static INumberLiteral Scan(TextInputRange input)
{
var chr = input.EndChar;
if (!IsDecimalDigit(chr))
{
return(null);
}
if (chr == '0')
{
var next = input.PeekChar();
switch (next)
{
case 'x':
case 'X':
input.Extend(nChars: 2);
return(ScanNumber(input, radix: 16, isDigit: IsHexDigit));
case 'o':
case 'O':
input.Extend(nChars: 2);
return(ScanNumber(input, radix: 8, isDigit: IsOctalDigit));
case 'b':
case 'B':
input.Extend(nChars: 2);
return(ScanNumber(input, radix: 2, isDigit: IsBinaryDigit));
}
}
return(ScanDecimalNumber(input));
}
static bool HandleEscape(TextInputRange input, StringBuilder result)
{
input.Extend();
var chr = input.EndChar;
switch (chr)
{
case '\0':
return(false);
case 't':
result.Append(value: '\t');
break;
case 'r':
result.Append(value: '\r');
break;
case 'n':
result.Append(value: '\n');
break;
// TODO: add Unicode escape handling
default:
result.Append(chr);
break;
}
input.Extend();
return(true);
}
static TokenData ScanSingleChar(TextInputRange input, Token token)
{
input.Extend();
return(new TokenData {
Range = input.Clone(), Type = token
});
}
static TokenData ScanNewLine(TextInputRange input)
{
input.NewLine();
input.ExtendWhitespaces();
return(new TokenData {
Range = input.Clone(), Type = Token.NewLineIndentation
});
}
static TokenData ScanWhitespaces(TextInputRange input)
{
input.Extend();
input.ExtendWhitespaces();
return(new TokenData {
Range = input.Clone(), Type = Token.WhiteSpaceSeperator
});
}
static TokenData ScanNumberLiteral(TextInputRange input)
{
var literal = NumberLiteralScanner.Scan(input);
Debug.Assert(literal != null); // TODO: Create error token
return(new TokenData {
Range = literal.Range, Type = Token.NumberLiteral, Data = literal
});
}
static TokenData ScanComment(TextInputRange input)
{
var comment = CommentScanner.Scan(input);
Debug.Assert(comment);
return(new TokenData {
Range = input.Clone(), Type = Token.Comment
});
}
// Scan basic double quoted strings
public static IStringLiteral Scan(TextInputRange input)
{
var chr = input.EndChar;
if (!IsDoubleQuote(chr))
{
return(null);
}
input.Extend();
var result = new StringBuilder();
while (true)
{
chr = input.EndChar;
if (chr == '\0')
{
return(null); // file end or invalid input
}
if (IsDoubleQuote(chr))
{
break;
}
if (input.IsEndNewline)
{
input.NewLine();
}
else
{
if (!IsTab(chr) && char.IsControl(chr))
{
HandleControl(input);
}
else
{
if (IsBackslash(chr))
{
if (!HandleEscape(input, result))
{
return(null);
}
}
else
{
result.Append(chr);
input.Extend();
}
}
}
}
input.Extend();
return(new StringLiteral {
Content = result.ToString(), Range = input.Clone()
});
}
public void PeekCharEndTest()
{
var input = new TextInputRange {
File = new TextFile {
Content = "A",
Filename = ""
}
};
Assert.AreEqual(expected: '\0', actual: input.PeekChar());
}
public void EndStringTest()
{
var input = new TextInputRange {
File = new TextFile {
Content = "ABC",
Filename = ""
}
};
var s = input.EndString(nChars: 4);
Assert.AreEqual(expected: "ABC", actual: s);
}
public void IsNotKeywordTest()
{
var input = new TextInputRange {
File = new TextFile {
Content = "ABCD",
Filename = ""
}
};
var s = input.IsKeyword(word: "ABC");
Assert.AreEqual(expected: false, actual: s);
}
public void SkipTest()
{
var input = new TextInputRange {
File = new TextFile {
Content = "ABC ",
Filename = ""
}
};
input.Skip(nChars: 2);
Assert.AreEqual(expected: 'C', actual: input.EndChar);
}
public void ExtendTest()
{
var input = new TextInputRange {
File = new TextFile {
Content = "ABC ",
Filename = ""
}
};
input.Extend(nChars: 2);
Assert.AreEqual(expected: "AB", actual: input.Text);
Assert.AreEqual(expected: 3, actual: input.End.Column);
}
public void TestNegativeBlockComments(string content)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var exception = Assert.Throws <Exception>(() => CommentScanner.Scan(input));
Assert.True(exception.Message.Equals(value: "Line Comment not Escaped."));
}
public void ScanFailures(string content)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var result = NumberLiteralScanner.Scan(input);
Assert.IsNull(result);
}
public void PeekCharStartTest()
{
var input = new TextInputRange {
File = new TextFile {
Content = "AB",
Filename = ""
}
};
var t = input.PeekChar();
Assert.AreEqual(expected: 'B', actual: t);
}
[TestCase(arg: "(+ )", TestName = "space is hard separator")] // closing bracket is missing, no entry left
public void ScanNewFailure(string content)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var result = OperatorScanner.Scan(input);
Assert.IsNull(result);
}
public void TestNegativeLineComments(string content)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var output = CommentScanner.Scan(input);
Assert.IsFalse(output);
Assert.AreEqual(content[index: 0], input.EndChar);
}
public void TestPositiveLineComments(string content)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var output = CommentScanner.Scan(input);
Assert.IsTrue(output);
Assert.IsFalse(input.IsEndValid);
}
/**
* # single line
*
* #<optional>#
* block comments
* #<optional>#
*/
public static bool Scan(TextInputRange range)
{
if (range.EndChar != '#')
{
return(false);
}
do
{
range.Extend();
} while (range.IsEndValid && !range.IsEndNewline && !range.IsEndWhitespace && range.EndChar != '#');
// single line comment
if (range.IsEndNewline)
{
return(true);
}
// is block comment
if (range.EndChar == '#')
{
range.Extend();
var commentMarker = range.Text;
while (range.IsEndValid && range.EndString(commentMarker.Length) != commentMarker)
{
if (range.IsEndNewline)
{
range.NewLine();
}
else
{
range.Extend();
}
}
if (range.EndString(commentMarker.Length) != commentMarker)
{
throw new Exception(message: "Line Comment not Escaped.");
}
range.Extend(commentMarker.Length);
}
else
{
do
{
range.Extend();
} while (range.IsEndValid && !range.IsEndNewline);
}
return(true);
}
public void ScanSuccess(string content, string output)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var result = StringLiteralScanner.Scan(input);
Assert.IsNotNull(result);
Assert.AreEqual(output, result.Content);
}
public void CollapseWhitespacesTest()
{
var input = new TextInputRange {
File = new TextFile {
Content = "AB C",
Filename = ""
}
};
input.Extend(nChars: 2);
input.CollapseWhitespaces();
Assert.AreEqual(expected: 'C', actual: input.EndChar);
Assert.AreEqual(expected: "", actual: input.Text);
}
public void ScanNewSuccess(string content, string id)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var result = OperatorScanner.Scan(input);
Assert.IsNotNull(result);
Assert.AreEqual(id, result.Content);
}
static bool IsStart(TextInputRange input)
{
var chr = input.EndChar;
if (chr == '.')
{
input.Extend(); // dot is allowed at start but not later
chr = input.EndChar;
}
if ('0' <= chr && '9' >= chr)
{
return(false); // digits are not allowed at start
}
return(IsContinue(input)); // no more special rules
}
public void TestNegativeLineEndLineComments(string content)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var output = CommentScanner.Scan(input);
Assert.IsTrue(output);
Assert.IsTrue(input.IsEndValid);
Assert.AreEqual(expected: '\n', actual: input.EndChar);
}
public void TestPositiveBlockComments(string content, int line)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var output = CommentScanner.Scan(input);
Assert.IsTrue(output);
Assert.IsFalse(input.IsEndValid);
Assert.AreEqual(line, input.End.Line);
}
// this will scan the longest valid regular entry
public static IIdentifierLiteral Scan(TextInputRange input)
{
if (!IsStart(input))
{
input.Backtrack(); // dot might have been skipped
return(null);
}
do
{
input.Extend();
} while (IsContinue(input));
return(new IdentifierLiteral {
Content = input.Text, Range = input.Clone()
});
}
public void ScanSuccess(string content, int radix, string integerPart, string fractionalPart, string exponentPart)
{
var input = new TextInputRange {
File = new TextFile {
Content = content,
Filename = ""
}
};
var result = NumberLiteralScanner.Scan(input);
Assert.IsNotNull(result);
Assert.AreEqual(radix, result.Radix);
Assert.AreEqual(integerPart, result.IntegerPart);
Assert.AreEqual(fractionalPart, result.FractionalPart);
Assert.AreEqual(exponentPart, result.ExponentPart);
}
static bool IsContinue(TextInputRange input)
{
var chr = input.EndChar;
switch (CharUnicodeInfo.GetUnicodeCategory(chr))
{
case UnicodeCategory.LowercaseLetter: // lower
case UnicodeCategory.UppercaseLetter: // UPPER
case UnicodeCategory.DecimalDigitNumber: // 0-9
case UnicodeCategory.ConnectorPunctuation: // various underscores _‿⁀⁔＿
return(true);
case UnicodeCategory.TitlecaseLetter: // Ligatures Dž, Lj, Nj and Dz
case UnicodeCategory.OtherLetter: // Hebrew etc. א
case UnicodeCategory.MathSymbol: // +=-
case UnicodeCategory.OtherSymbol: // ©®⌛⌚
case UnicodeCategory.DashPunctuation: // all kinds of hyphens
case UnicodeCategory.LetterNumber: // Roman Numbers (look like normal letters)
case UnicodeCategory.OtherNumber: // ½²
case UnicodeCategory.PrivateUse: // emojis etc.
case UnicodeCategory.CurrencySymbol: // ¢¥$€
case UnicodeCategory.OtherPunctuation: // like !?#.
case UnicodeCategory.OpenPunctuation: // [{(
case UnicodeCategory.InitialQuotePunctuation: // “«
case UnicodeCategory.ClosePunctuation: // ]})
case UnicodeCategory.FinalQuotePunctuation: // ”»
case UnicodeCategory.ModifierLetter:
case UnicodeCategory.NonSpacingMark:
case UnicodeCategory.SpacingCombiningMark:
case UnicodeCategory.EnclosingMark:
case UnicodeCategory.SpaceSeparator:
case UnicodeCategory.LineSeparator:
case UnicodeCategory.ParagraphSeparator:
case UnicodeCategory.Control:
case UnicodeCategory.Format:
case UnicodeCategory.Surrogate:
case UnicodeCategory.ModifierSymbol:
case UnicodeCategory.OtherNotAssigned:
return(false);
default:
throw new ArgumentOutOfRangeException();
}
}
static void HandleControl(TextInputRange input)
{
// do not add arbitrary control characters to internal strings.
input.Extend();
}
