/// <summary> /// Classify the given LexToken into a ClassificationTag. /// </summary> /// <param name="token">The token to be classified.</param> /// <returns>The smState value.</returns> public ClassificationTag Classify(LexToken token) { int classTag; UiExceptionHelper.CheckNotNull(token, "token"); classTag = AcceptLexToken(token); if (classTag == SMSTATE_CODE && _keywords.ContainsKey(token.Text)) { return(ClassificationTag.Keyword); } // Parsing a token whoose Text value is set to '\' // causes the classifier to set/reset is escaping mode. if (token.Text == "\\" && _sm_output == SMSTATE_STRING && !_escaping) { _escaping = true; } else { _escaping = false; } return(_tags[classTag]); }
/// <summary> /// Classify the given token and get its corresponding SMSTATE value. /// </summary> /// <param name="token">The LexToken to be classified.</param> /// <returns>An SMSTATE value.</returns> protected int AcceptLexToken(LexToken token) { int smState; if (_escaping) { return(SMSTATE_STRING); } smState = GetTokenSMSTATE(_sm_output, token.Tag); _sm_output = GetSMSTATE(_sm_output, token.Tag); return(smState); }
/// <summary> /// Builds the list of all LexToken which text value starts with the one in starter. /// </summary> /// <param name="starter">The token that the reference text.</param> /// <param name="output">The list of tokens which text starts with the one in starter.</param> protected void PopulateTokenStartingWith(LexToken starter, List <LexToken> output) { InternalLexToken token; UiExceptionHelper.CheckNotNull(starter, "starter"); UiExceptionHelper.CheckNotNull(output, "output"); output.Add(starter); token = (InternalLexToken)starter; foreach (LexToken item in token.StartingWith) { output.Add(item); } return; }
private void _checkOutput(string sequence, LexToken[] expected) { Lexer lexer; StringBuilder recognized; string error; int i; int j; lexer = new Lexer(); lexer.Parse(sequence); recognized = new StringBuilder(); i = 0; while (lexer.Next()) { recognized.Append(lexer.CurrentToken.Text); error = String.Format("Token [{0}] was expected, but lexer returned [{1}] instead, near: [{2}].", expected[i], lexer.CurrentToken, recognized.ToString()); Assert.That(lexer.CurrentToken, Is.EqualTo(expected[i]), error); i++; } Assert.That(lexer.Next(), Is.False, "Error, there are unvisited tokens left."); error = "missing "; j = i; while (j < expected.Length) { error += expected[j].ToString(); error += ", "; ++j; } Assert.That(i == expected.Length, "Error, more tokens were expected. {0}", error); return; }
public new int AcceptLexToken(LexToken token) { return (base.AcceptLexToken(token)); }
/// <summary> /// Classify the given token and get its corresponding SMSTATE value. /// </summary> /// <param name="token">The LexToken to be classified.</param> /// <returns>An SMSTATE value.</returns> protected int AcceptLexToken(LexToken token) { int smState; if (_escaping) return (SMSTATE_STRING); smState = GetTokenSMSTATE(_sm_output, token.Tag); _sm_output = GetSMSTATE(_sm_output, token.Tag); return (smState); }
/// <summary> /// Classify the given LexToken into a ClassificationTag. /// </summary> /// <param name="token">The token to be classified.</param> /// <returns>The smState value.</returns> public ClassificationTag Classify(LexToken token) { int classTag; UiExceptionHelper.CheckNotNull(token, "token"); classTag = AcceptLexToken(token); if (classTag == SMSTATE_CODE && _keywords.ContainsKey(token.Text)) return (ClassificationTag.Keyword); // Parsing a token whoose Text value is set to '\' // causes the classifier to set/reset is escaping mode. if (token.Text == "\\" && _sm_output == SMSTATE_STRING && !_escaping) _escaping = true; else _escaping = false; return (_tags[classTag]); }
/// <summary> /// Builds the list of all LexToken which text value starts with the one in starter. /// </summary> /// <param name="starter">The token that the reference text.</param> /// <param name="output">The list of tokens which text starts with the one in starter.</param> protected void PopulateTokenStartingWith(LexToken starter, List<LexToken> output) { InternalLexToken token; UiExceptionHelper.CheckNotNull(starter, "starter"); UiExceptionHelper.CheckNotNull(output, "output"); output.Add(starter); token = (InternalLexToken)starter; foreach (LexToken item in token.StartingWith) output.Add(item); return; }
public new void PopulateTokenStartingWith(LexToken starter, List<LexToken> output) { base.PopulateTokenStartingWith(starter, output); }