/// <summary>Gets the original source text for a token if available, under the
/// assumption that the specified source file correctly specifies where the
/// token came from. If the token is synthetic, returns <see cref="UString.Null"/>.</summary>
public UString SourceText(ICharSource chars)
{
if ((uint)StartIndex <= (uint)chars.Count)
{
return(chars.Slice(StartIndex, Length));
}
return(UString.Null);
}
/// <summary>Gets the original source text for a token if available, under the
/// assumption that the specified source file correctly specifies where the
/// token came from. If the token is synthetic, returns <see cref="UString.Null"/>.</summary>
public UString SourceText(ICharSource file)
{
if ((uint)StartIndex <= (uint)file.Count)
{
return(file.Slice(StartIndex, Length));
}
return(UString.Null);
}
/// <summary>Helps get the "text value" from tokens that used one of the
/// constructors designed to support this use case, e.g.
/// <see cref="Token(int, int, UString, NodeStyle, Symbol, int, int)"/>.
/// If one of the other constructors was used, this function returns the same
/// value as <see cref="SourceText(ICharSource)"/>.</summary>
/// <param name="source">Original source code or lexer from which this token was derived.</param>
public UString TextValue(ICharSource source)
{
if (SubstringOffset == 0xFF)
{
return(((Tuple <Symbol, UString>)_value).Item2);
}
return(source.Slice(StartIndex + SubstringOffset, Length - SubstringOffset - SubstringOffsetFromEnd));
}
/// <summary>Expresses a token as a string, using LES printers for identifiers and literals.</summary>
/// <remarks>Note that some Tokens do not contain enough information to
/// reconstruct a useful token string, e.g. comment tokens do not store the
/// comment but merely contain the location of the comment in the source code.
/// For performance reasons, a <see cref="Token"/> does not have a reference
/// to its source file, so this method cannot return the original string.
/// </remarks>
public static string ToString(Token t, ICharSource sourceCode)
{
if (sourceCode != null && t.EndIndex <= sourceCode.Count)
{
return(sourceCode.Slice(t.StartIndex, t.Length).ToString());
}
StringBuilder sb = new StringBuilder();
switch (t.Kind)
{
case TokenKind.Spaces: return((t.Value ?? " ").ToString());
case TokenKind.Comment:
if (t.Type() == TokenType.SLComment)
{
return("// (comment)");
}
else
{
return("/* (comment) */");
}
case TokenKind.Id:
return(Les2Printer.PrintId(t.Value as Symbol ?? GSymbol.Empty));
case TokenKind.Literal:
return(Les2Printer.PrintLiteral(t.Value, t.Style));
}
if (t.Value != null)
{
if (t.Type() == TokenType.BQOperator)
{
return(Les2Printer.PrintString((t.Value ?? "").ToString(), '`', false));
}
else if (t.Type() == TokenType.Shebang)
{
return("#!" + t.Value.ToString() + "\n");
}
return(t.Value.ToString());
}
switch (t.Kind)
{
case TokenKind.LParen: return("(");
case TokenKind.RParen: return(")");
case TokenKind.LBrack: return("[");
case TokenKind.RBrack: return("]");
case TokenKind.LBrace: return("{");
case TokenKind.RBrace: return("}");
case TokenKind.Indent: return("(Indent)");
case TokenKind.Dedent: return("(Dedent)");
case TokenKind.Dot: return("(Dot)");
case TokenKind.Assignment: return("(Assignment)");
case TokenKind.Operator: return("(Operator)");
case TokenKind.Separator: return("(Separator)");
case TokenKind.AttrKeyword: return("(AttrKeyword)");
case TokenKind.TypeKeyword: return("(TypeKeyword)");
case TokenKind.OtherKeyword: return("(OtherKeyword)");
}
return("(Type " + t.TypeInt + ")");
}
/// <summary>Expresses an EC# token as a string.</summary>
/// <remarks>Note that some Tokens do not contain enough information to
/// reconstruct a useful token string, e.g. comment tokens do not store the
/// comment but merely contain the location of the comment in the source code.
/// For performance reasons, a <see cref="Token"/> does not have a reference
/// to its source file, so this method cannot return the original string.
/// <para/>
/// The results are undefined if the token was not produced by <see cref="EcsLexer"/>.
/// </remarks>
public static string ToString(Token t, ICharSource sourceCode)
{
if (sourceCode != null && t.EndIndex <= sourceCode.Count)
{
return(sourceCode.Slice(t.StartIndex, t.Length).ToString());
}
StringBuilder sb = new StringBuilder();
if (t.Kind == TokenKind.Operator || t.Kind == TokenKind.Assignment || t.Kind == TokenKind.Dot)
{
if (t.Type() == TT.BQString)
{
return(EcsNodePrinter.PrintString((t.Value ?? "").ToString(), '`', false));
}
string value = (t.Value ?? "(null)").ToString();
return(value);
}
switch (t.Type())
{
case TT.EOF: return("/*EOF*/");
case TT.Spaces: return(" ");
case TT.Newline: return("\n");
case TT.SLComment: return("//\n");
case TT.MLComment: return("/**/");
case TT.Shebang: return("#!" + (t.Value ?? "").ToString() + "\n");
case TT.Id:
case TT.ContextualKeyword:
case TT.LinqKeyword:
var mode = (t.Style & NodeStyle.Operator) != 0 ? EcsNodePrinter.IdPrintMode.Operator :
(t.Style & NodeStyle.VerbatimId) != 0 ? EcsNodePrinter.IdPrintMode.Verbatim : EcsNodePrinter.IdPrintMode.Normal;
return(EcsNodePrinter.PrintId(t.Value as Symbol ?? GSymbol.Empty, mode));
case TT.Base: return("base");
case TT.This: return("this");
case TT.Literal:
return(EcsNodePrinter.PrintLiteral(t.Value, t.Style));
case TT.Comma: return(",");
case TT.Semicolon: return(";");
case TT.LParen: return("(");
case TT.RParen: return(")");
case TT.LBrack: return("[");
case TT.RBrack: return("]");
case TT.LBrace: return("{");
case TT.RBrace: return("}");
case TT.AttrKeyword:
string value = (t.Value ?? "(null)").ToString();
return(value);
case TT.TypeKeyword:
Symbol valueSym = (t.Value as Symbol) ?? GSymbol.Empty;
string result;
if (EcsFacts.TypeKeywords.TryGetValue(valueSym, out result))
{
return(result);
}
else
{
Debug.Fail("Unexpected value for " + t.Type());
return((t.Value ?? "(null)").ToString());
}
case TT.CheckedOrUnchecked:
Debug.Assert(LNode.IsSpecialName((Symbol)t.Value));
return(((Symbol)t.Value).Name.Substring(1));
case TT.Break: return("break");
case TT.Case: return("case");
case TT.Class: return("class");
case TT.Continue: return("continue");
case TT.Default: return("default");
case TT.Delegate: return("delegate");
case TT.Do: return("do");
case TT.Enum: return("enum");
case TT.Event: return("event");
case TT.Fixed: return("fixed");
case TT.For: return("for");
case TT.Foreach: return("foreach");
case TT.Goto: return("goto");
case TT.If: return("if");
case TT.Interface: return("interface");
case TT.Lock: return("lock");
case TT.Namespace: return("namespace");
case TT.Return: return("return");
case TT.Struct: return("struct");
case TT.Switch: return("switch");
case TT.Throw: return("throw");
case TT.Try: return("try");
case TT.Using: return("using");
case TT.While: return("while");
case TT.Operator: return("operator");
case TT.Sizeof: return("sizeof");
case TT.Typeof: return("typeof");
case TT.Else: return("else");
case TT.Catch: return("catch");
case TT.Finally: return("finally");
case TT.In: return("in");
case TT.As: return("as");
case TT.Is: return("is");
case TT.New: return("new");
case TT.Out: return("out");
case TT.Stackalloc: return("stackalloc");
case TT.PPif: return("#if");
case TT.PPelse: return("#else");
case TT.PPelif: return("#elif");
case TT.PPendif: return("#endif");
case TT.PPdefine: return("#define");
case TT.PPundef: return("#undef");
case TT.PPwarning: return("#warning" + t.Value);
case TT.PPerror: return("#error" + t.Value);
case TT.PPnote: return("#note" + t.Value);
case TT.PPline: return("#line");
case TT.PPregion: return("#region" + t.Value);
case TT.PPendregion: return("#endregion");
case TT.PPpragma: return("#pragma");
case TT.PPignored: return((t.Value ?? "").ToString());
default:
return(string.Format("@`unknown token 0x{0:X4}`", t.TypeInt));
}
}
private void ReadBlock()
{
_block = _source.Slice(_inputPosition, CachedBlockSize);
_blockStart = _inputPosition;
LA0 = _block[0, -1];
}
