/// <summary>
/// Lex an identifier.
/// </summary>
private Token LexIdent()
{
bool fVerbatim = false;
if (ChCur == '@')
{
fVerbatim = true;
ChNext();
}
NormStr nstr = LexIdentCore(ref fVerbatim);
if (nstr == null)
{
// Error already reported.
return(null);
}
if (!fVerbatim)
{
KeyWordTable.KeyWordKind kind;
if (_lex._kwt.IsKeyWord(nstr, out kind))
{
return(KeyToken.CreateKeyWord(GetSpan(), nstr.Value.ToString(), kind.Kind, kind.IsContextKeyWord));
}
}
return(new IdentToken(GetSpan(), nstr.Value.ToString()));
}
public bool GetWordVector(ref ReadOnlyMemory <char> word, float[] wordVector)
{
NormStr str = _pool.Get(word);
if (str != null)
{
WordVectors.CopyTo(str.Id * Dimension, wordVector, Dimension);
return(true);
}
return(false);
}
public bool GetWordVector(ref DvText word, float[] wordVector)
{
if (word.IsNA)
{
return(false);
}
string rawWord = word.GetRawUnderlyingBufferInfo(out int ichMin, out int ichLim);
NormStr str = _pool.Get(rawWord, ichMin, ichLim);
if (str != null)
{
_wordVectors.CopyTo(str.Id * Dimension, wordVector, Dimension);
return(true);
}
return(false);
}
/// <summary>
/// Called to lex a punctuator (operator). Asserts the current character lex type
/// is LexCharType.Punc.
/// </summary>
private Token LexPunc()
{
int cchPunc = 0;
TokKind tidPunc = TokKind.None;
_sb.Length = 0;
_sb.Append(ChCur);
for (; ;)
{
TokKind tidCur;
NormStr nstr = _lex._pool.Add(_sb);
if (!_lex._kwt.IsPunctuator(nstr, out tidCur))
{
break;
}
if (tidCur != TokKind.None)
{
// This is a real punctuator, not just a prefix.
tidPunc = tidCur;
cchPunc = _sb.Length;
}
char ch = ChPeek(_sb.Length);
if (!LexCharUtils.IsPunc(ch))
{
break;
}
_sb.Append(ch);
}
if (cchPunc == 0)
{
return(LexError());
}
while (--cchPunc >= 0)
{
ChNext();
}
return(KeyToken.Create(GetSpan(), tidPunc));
}
public bool TryAddPunctuator(string str, TokKind tid)
{
Contracts.AssertNonEmpty(str);
// Note: this assumes that once a prefix is found, that all shorter
// prefixes are mapped to something (TokKind.None to indicate that
// it is only a prefix and not itself a token).
TokKind tidCur;
NormStr nstr = _pool.Add(str);
if (_mpnstrtidPunc.TryGetValue(_pool.Add(str), out tidCur))
{
if (tidCur == tid)
{
return(true);
}
if (tidCur != TokKind.None)
{
return(false);
}
}
else
{
// Map all prefixes (that aren't already mapped) to TokKind.None.
for (int cch = str.Length; --cch > 0;)
{
NormStr nstrTmp = _pool.Add(str.Substring(0, cch));
TokKind tidTmp;
if (_mpnstrtidPunc.TryGetValue(_pool.Add(nstrTmp.Value), out tidTmp))
{
break;
}
_mpnstrtidPunc.Add(nstrTmp, TokKind.None);
}
}
_mpnstrtidPunc[nstr] = tid;
return(true);
}
public bool IsKeyWord(NormStr nstr, out KeyWordKind kind)
{
Contracts.Assert(!nstr.Value.IsEmpty);
return(_mpnstrtidWord.TryGetValue(nstr, out kind));
}
public bool IsPunctuator(NormStr nstr, out TokKind tid)
{
Contracts.Assert(!nstr.Value.IsEmpty);
return(_mpnstrtidPunc.TryGetValue(nstr, out tid));
}
/// <summary>
/// Lex a comment.
/// </summary>
private Token LexComment()
{
Contracts.Assert(ChCur == '/');
int ichErr = _cursor.IchCur;
switch (ChPeek(1))
{
default:
return(LexPunc());
case '/':
// Single line comment.
ChNext();
_sb.Length = 0;
_sb.Append("//");
for (; ;)
{
if (LexCharUtils.IsLineTerm(ChNext()) || Eof)
{
return(new CommentToken(GetSpan(), _sb.ToString(), 0));
}
_sb.Append(ChCur);
}
case '*':
/* block comment */
ChNext();
_sb.Length = 0;
_sb.Append("/*");
ChNext();
int lines = 0;
for (; ;)
{
if (Eof)
{
ReportError(ichErr, _cursor.IchCur, ErrId.UnterminatedComment);
break;
}
char ch = ChCur;
if (LexCharUtils.IsLineTerm(ch))
{
ch = LexLineTerm(_sb);
lines++;
}
else
{
ChNext();
}
_sb.Append(ch);
if (ch == '*' && ChCur == '/')
{
_sb.Append('/');
ChNext();
break;
}
}
// We support comment keywords.
KeyWordTable.KeyWordKind kind;
NormStr nstr = _lex._pool.Add(_sb);
if (_lex._kwt.IsKeyWord(nstr, out kind))
{
return(KeyToken.CreateKeyWord(GetSpan(), nstr.ToString(), kind.Kind, kind.IsContextKeyWord));
}
return(new CommentToken(GetSpan(), _sb.ToString(), lines));
}
}