static bool IsSpecialIdentifier(Symbol name, out bool backquote)
{
backquote = name.Name.Length == 0;
bool special = false, first = true;
foreach (char c in name.Name)
{
if (!Les2Lexer.IsIdContChar(c))
{
if (Les2Lexer.IsSpecialIdChar(c))
{
special = true;
}
else
{
backquote = true;
}
}
else if (first && !Les2Lexer.IsIdStartChar(c))
{
special = true;
}
first = false;
}
// Watch out for @`-inf_d` and @`-inf_f`, because they will be
// interpreted as named literals if we don't backquote them.
if (special && !backquote && (name.Name == "-inf_d" || name.Name == "-inf_f"))
{
backquote = true;
}
return(special || backquote);
}
protected string ParseStringValue(bool parseNeeded, bool isTripleQuoted)
{
if (SkipValueParsing)
{
return(null);
}
string value;
if (parseNeeded)
{
UString original = CharSource.Slice(_startPosition, InputPosition - _startPosition);
value = Les2Lexer.UnescapeQuotedString(ref original, Error, IndentString, true);
Debug.Assert(original.IsEmpty);
}
else
{
Debug.Assert(CharSource.TryGet(InputPosition - 1, '?') == CharSource.TryGet(_startPosition, '!'));
if (isTripleQuoted)
{
value = CharSource.Slice(_startPosition + 3, InputPosition - _startPosition - 6).ToString();
}
else
{
value = CharSource.Slice(_startPosition + 1, InputPosition - _startPosition - 2).ToString();
}
}
return(value);
}
void Case(string input, TokenType[] tokenTypes, params object[] values)
{
Debug.Assert(values.Length <= tokenTypes.Length);
bool error = false;
var lexer = new Les2Lexer(input, new MessageSinkFromDelegate((type, ctx, msg, args) => {
TraceMessageSink.Value.Write(type, ctx, msg, args); error = true;
}));
int index = 0;
for (int i = 0; i < tokenTypes.Length; i++)
{
error = false;
Token token = lexer.NextToken().Value;
Assert.LessOrEqual(index, token.StartIndex);
Assert.AreEqual(tokenTypes[i], token.Type());
if (i < values.Length)
{
Assert.AreEqual(values[i] == (object)ERROR, error);
if (!error)
{
Assert.AreEqual(values[i], token.Value);
}
}
index = token.EndIndex;
}
var nothing = lexer.NextToken();
Assert.That(!nothing.HasValue);
}
static bool IsSpecialIdentifier(UString name, out bool backquote)
{
backquote = name.Length == 0;
bool special = false, first = true;
foreach (char c in name)
{
if (!Les2Lexer.IsIdContChar(c))
{
if (Les2Lexer.IsSpecialIdChar(c))
{
special = true;
}
else
{
backquote = true;
}
}
else if (first && !Les2Lexer.IsIdStartChar(c))
{
special = true;
}
first = false;
}
// Watch out for named literals with punctuation e.g. @`-inf_d` and @`-inf_f`:
// they will be interpreted as named literals if we don't backquote them.
if (special && !backquote && Les2Lexer.NamedLiterals.ContainsKey(name))
{
backquote = true;
}
return(special || backquote);
}
protected override void StartToken(char nextCh)
{
if (_newlinePending)
{
Newline();
}
if (Les2Lexer.IsIdContChar(_lastCh) && Les2Lexer.IsIdContChar(nextCh))
{
_out.Write(' ');
}
else if (Les2Lexer.IsOpContChar(_lastCh) && Les2Lexer.IsOpContChar(nextCh))
{
_out.Write(' ');
}
else if (_lastCh == '-' && (nextCh >= '0' && nextCh <= '9')) // - 2 is different from -2 (-(2) vs integer literal)
{
_out.Write(' ');
}
}
protected override void OnNodeChanged(char nextCh)
{
var lastCh = LastCharWritten;
if (Les2Lexer.IsIdContChar(lastCh) && Les2Lexer.IsIdContChar(nextCh))
{
StringBuilder.Append(' ');
}
else if (Les2Lexer.IsOpContChar(lastCh) && Les2Lexer.IsOpContChar(nextCh))
{
StringBuilder.Append(' ');
}
else if (JustWroteSymbolOrSpecialId && Les2Lexer.IsSpecialIdChar(nextCh))
{
StringBuilder.Append(' ');
}
JustWroteSymbolOrSpecialId = false;
}
protected object ParseSQStringValue(bool parseNeeded)
{
if (SkipValueParsing)
{
return(null);
}
else
{
var text = Text();
if (!parseNeeded && text.Length == 3)
{
return(text[1]);
}
else
{
var value = Les2Lexer.ParseSQStringValue(text, Error);
if (value is string)
{
return(new CustomLiteral(value, (Symbol)"char"));
}
return(value);
}
}
}
List<Token> Lex(string input, bool skipWS = true)
{
var lexer = new Les2Lexer(input, TraceMessageSink.Value);
var lexer2 = new TokensToTree(lexer, skipWS);
var list = new List<Token>();
Maybe<Token> token;
while ((token = lexer2.NextToken()).HasValue)
list.Add(token.Value);
return list;
}