protected void UnescapeSymbolValue()
{
Debug.Assert(CharSource[_startPosition] == '@' && CharSource[_startPosition + 1] == '@');
if (SkipValueParsing)
{
return;
}
_value = _s;
UString original = CharSource.Slice(_startPosition + 2, InputPosition - _startPosition - 2);
if (_hasEscapes)
{
_textValue = Les3Lexer.UnescapeQuotedString(ref original, Error);
Debug.Assert(original.IsEmpty);
}
else if (original[0, '\0'] == '`')
{
_textValue = original.Substring(1, original.Length - 2);
}
else
{
_textValue = original;
}
}
protected UString UnescapeString(bool isTripleQuoted, bool allowExtraIndent = false)
{
if (SkipValueParsing)
{
return("");
}
if (_hasEscapes)
{
UString original = CharSource.Slice(_startPosition, InputPosition - _startPosition);
_textValue = Les3Lexer.UnescapeQuotedString(ref original, Error, IndentString, allowExtraIndent);
Debug.Assert(original.IsEmpty);
}
else
{
Debug.Assert(CharSource.TryGet(InputPosition - 1, '?') == CharSource.TryGet(_startPosition, '!'));
if (isTripleQuoted)
{
_textValue = CharSource.Slice(_startPosition + 3, InputPosition - _startPosition - 6).ToString();
}
else
{
_textValue = CharSource.Slice(_startPosition + 1, InputPosition - _startPosition - 2).ToString();
}
}
return(_textValue);
}
// After the generated lexer code determines the boundaries of the token,
// one of these methods extracts the value of the token (e.g. "17L" => (long)17)
// There are value parsers for identifiers, numbers, and strings; certain
// parser cores are also accessible as public static methods.
#region String unescaping (including public UnescapeQuotedString())
protected void UnescapeSQStringValue()
{
_value = _c;
var text = Text();
if (!_hasEscapes)
{
_textValue = text.Slice(1, text.Length - 2);
}
else
{
_textValue = Les3Lexer.UnescapeQuotedString(ref text, Error);
}
}
void Case(UString input, TokenType[] tokenTypes, params object[] expected)
{
Debug.Assert(expected.Length <= tokenTypes.Length);
bool error = false;
var lexer = new Les3Lexer(input, "", new MessageSinkFromDelegate((type, ctx, msg, args) => {
TraceMessageSink.Value.Write(type, ctx, msg, args); error = true;
}));
int index = 0;
for (int i = 0; i < tokenTypes.Length; i++)
{
error = false;
Token token = lexer.NextToken().Value;
Assert.LessOrEqual(index, token.StartIndex);
Assert.AreEqual(tokenTypes[i], token.Type());
if (i < expected.Length)
{
var value = token.IsUninterpretedLiteral ? token.TextValue(lexer).ToString() : token.Value;
var expected_i = expected[i];
if (expected[i] is Error e)
{
Assert.IsTrue(error, "Expected error didn't occur in «{0}»", input);
expected_i = e.Value;
}
else
{
Assert.IsFalse(error, "Unexpected error in token [{0}] of «{1}»", i, input);
}
if (expected_i is Pair <string, Symbol> pair)
{
Assert.AreEqual(pair.A, value);
Assert.AreEqual(pair.B, token.TypeMarker);
}
else
{
Assert.AreEqual(expected_i, value);
}
}
index = token.EndIndex;
}
var nothing = lexer.NextToken();
Assert.That(!nothing.HasValue, "Extra token after the expected ones in «" + input.ToString() + "»");
}
/// <summary>Parses an LES2-style identifier such as <c>foo</c>, <c>@foo</c>,
/// <c>@`foo`</c> or <c>@--punctuation--</c>.
/// </summary>
/// <param name="source">Text to parse. On return, the range has been
/// decreased by the length of the token; this method also stops if this
/// range becomes empty.</param>
/// <param name="onError">A method to call on error</param>
/// <param name="checkForNamedLiteral">This is set to true when the input
/// starts with @ but doesn't use backquotes, which could indicate that
/// it is an LES named literal such as @false or @null.</param>
/// <returns>The parsed version of the identifier.</returns>
public static string ParseIdentifier(ref UString source, Action <int, string> onError, out bool checkForNamedLiteral)
{
checkForNamedLiteral = false;
StringBuilder parsed = TempSB();
UString start = source;
bool fail;
int c = source.PopFirst(out fail);
if (c == '@')
{
// expecting: (BQString | Star(Set("[0-9a-zA-Z_'#~!%^&*-+=|<>/?:.@$]") | IdExtLetter))
c = source.PopFirst(out fail);
if (c == '`')
{
Les3Lexer.UnescapeString(ref source, (char)c, false, onError, parsed);
}
else
{
while (SpecialIdSet.Contains(c) || c >= 128 && char.IsLetter((char)c))
{
parsed.Append((char)c);
c = source.PopFirst(out fail);
}
checkForNamedLiteral = true;
}
}
else if (IsIdStartChar(c))
{
parsed.Append(c);
for (;;)
{
c = source.PopFirst(out fail);
if (!IsIdContChar(c))
{
break;
}
parsed.Append((char)c);
}
}
return(parsed.ToString());
}
protected internal static object ParseSQStringValue(UString text, Action <int, string> Error)
{
var sb = TempSB();
Les3Lexer.UnescapeQuotedString(ref text, Error, sb, "\t");
Debug.Assert(text.IsEmpty);
if (sb.Length == 1)
{
return(CG.Cache(sb[0]));
}
else
{
if (sb.Length == 0)
{
Error(0, Localize.Localized("Empty character literal"));
}
else
{
Error(0, Localize.Localized("Character literal has {0} characters (there should be exactly one)", sb.Length));
}
return(sb.ToString());
}
}
void Case(UString input, TokenType[] tokenTypes, params object[] expected)
{
Debug.Assert(expected.Length <= tokenTypes.Length);
bool error = false;
var lexer = new Les3Lexer(input, "", new MessageSinkFromDelegate((type, ctx, msg, args) => {
TraceMessageSink.Value.Write(type, ctx, msg, args); error = true;
}));
int index = 0;
for (int i = 0; i < tokenTypes.Length; i++)
{
error = false;
Token token = lexer.NextToken().Value;
Assert.LessOrEqual(index, token.StartIndex);
Assert.AreEqual(tokenTypes[i], token.Type());
if (i < expected.Length)
{
if (expected[i] is Error)
{
Assert.IsTrue(error, "Expected error didn't occur in «{0}»", input);
Assert.AreEqual(((Error)expected[i]).Value, token.Value);
}
else
{
Assert.AreEqual(expected[i], token.Value);
Assert.IsFalse(error, "Unexpected error in token [{0}] of «{1}»", i, input);
}
}
index = token.EndIndex;
}
var nothing = lexer.NextToken();
Assert.That(!nothing.HasValue, "Extra token after the expected ones in «" + input.ToString() + "»");
}
public static bool UnescapeString(ref UString sourceText, char quoteType, bool isTripleQuoted, Action <int, string> onError, StringBuilder sb, UString indentation = default(UString), bool les3TQIndents = false) => Les3Lexer.UnescapeString(ref sourceText, quoteType, isTripleQuoted, onError, sb, indentation, les3TQIndents);
public static void UnescapeQuotedString(ref UString sourceText, Action <int, string> onError, StringBuilder sb, UString indentation = default(UString), bool les3TQIndents = false) => Les3Lexer.UnescapeQuotedString(ref sourceText, onError, sb, indentation, les3TQIndents);