public void SetUp()
{
Lexer lexer;
_emptyDictionary = new TestingTokenDictionary();
lexer = new Lexer();
_filledDictionary = lexer.Dictionary;
Assert.That(_filledDictionary, Is.Not.Null);
Assert.That(_filledDictionary.Count, Is.GreaterThan(0));
return;
}
private void _checkOutput(string sequence, LexToken[] expected)
{
Lexer lexer;
StringBuilder recognized;
string error;
int i;
int j;
lexer = new Lexer();
lexer.Parse(sequence);
recognized = new StringBuilder();
i = 0;
while (lexer.Next())
{
recognized.Append(lexer.CurrentToken.Text);
error = String.Format("Token [{0}] was expected, but lexer returned [{1}] instead, near: [{2}].",
expected[i],
lexer.CurrentToken,
recognized.ToString());
Assert.That(lexer.CurrentToken, Is.EqualTo(expected[i]), error);
i++;
}
Assert.That(lexer.Next(), Is.False, "Error, there are unvisited tokens left.");
error = "missing ";
j = i;
while (j < expected.Length)
{
error += expected[j].ToString();
error += ", ";
++j;
}
Assert.That(i == expected.Length, "Error, more tokens were expected. {0}", error);
return;
}
private void _checkClassification(string sequence, Couple[] expected)
{
Lexer lexer;
ClassificationTag returned_tag;
TestingClassifier classifier;
StringBuilder recognized;
string error;
int i;
classifier = new TestingClassifier();
lexer = new Lexer();
lexer.Parse(sequence);
recognized = new StringBuilder();
i = 0;
while (lexer.Next())
{
recognized.Append(lexer.CurrentToken.Text);
error = String.Format(
"Token [{0}] was expected, but [{1}] was returned instead, near: [{2}].",
expected[i].Text,
lexer.CurrentToken.Text,
recognized.ToString());
Assert.That(lexer.CurrentToken.Text, Is.EqualTo(expected[i].Text), error);
returned_tag = classifier.Classify(lexer.CurrentToken);
error = String.Format(
"ClassificationTag [{0}] was expected, but [{1}] was returned instead, near: [{2}].",
expected[i].Value,
returned_tag,
recognized.ToString());
Assert.That(returned_tag, Is.EqualTo(expected[i].Value), error);
i++;
}
Assert.That(lexer.Next(), Is.False, "Error, there are unvisited tokens left.");
Assert.That(i == expected.Length, "Error, more tokens were expected.");
return;
}
public void Test_Escaping_sequence()
{
Lexer _lexer;
_lexer = new Lexer();
// this ensure that escaping can be set in string context only
_lexer.Parse("\\\\");
_classifier.Reset();
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.Code));
Assert.That(_classifier.Escaping, Is.False);
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.Code));
Assert.That(_classifier.Escaping, Is.False);
// this ensure that parsing "\\\\" two times
// set and unset Escaping flag correctly
_lexer.Parse("\"\\\\\"");
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_classifier.Escaping, Is.False);
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_classifier.Escaping, Is.True);
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_classifier.Escaping, Is.False);
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_classifier.Escaping, Is.False);
// this ensure that first 'a' is considered as string, second as code
_lexer.Parse("\"\\\"a\"a");
_classifier.Reset();
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_classifier.Escaping, Is.True);
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_classifier.Escaping, Is.False);
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("a"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("a"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.Code));
// another test, this time 'a' should be considered as code
_lexer.Parse("\"\\\\\"a\"");
_classifier.Reset();
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
Assert.That(_lexer.Next(), Is.True);
Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("a"));
Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.Code));
// this ensure that Reset() reset escaping to false
_lexer.Parse("\"\\");
_lexer.Next();
_classifier.Classify(_lexer.CurrentToken);
_lexer.Next();
_classifier.Classify(_lexer.CurrentToken);
Assert.That(_classifier.Escaping, Is.True);
_classifier.Reset();
Assert.That(_classifier.Escaping, Is.False);
return;
}
public void Test_Reset()
{
Lexer lexer;
lexer = new Lexer();
lexer.Parse("/*int");
lexer.Next();
_classifier.Classify(lexer.CurrentToken);
_classifier.Reset();
lexer.Next();
Assert.That(_classifier.Classify(lexer.CurrentToken), Is.EqualTo(ClassificationTag.Keyword));
return;
}
public void Test_Classify_As_Keyword()
{
TokenClassifier classifier;
ClassificationTag result;
string error;
Lexer lexer;
lexer = new Lexer();
lexer.Parse(
"abstract event new struct as explicit null switch " +
"base extern object this bool false operator throw " +
"break finally out true byte fixed override try case " +
"float params typeof catch for private uint char " +
"foreach protected ulong checked goto public unchecked " +
"class if readonly unsafe const implicit ref ushort " +
"continue in return using decimal int sbyte virtual " +
"default interface sealed volatile delegate internal " +
"short void do is sizeof while double lock stackalloc " +
"else long static enum namespace string get set region " +
"endregion ");
classifier = new TokenClassifier();
while (lexer.Next())
{
if (lexer.CurrentToken.Text.Trim() == "")
continue;
result = classifier.Classify(lexer.CurrentToken);
error = String.Format("Classification: [{0}] was expected for token [{1}] but [{2}] was returned.",
ClassificationTag.Keyword,
lexer.CurrentToken,
result);
Assert.That(
result,
Is.EqualTo(ClassificationTag.Keyword),
error);
}
return;
}
/// <summary>
/// Analyzes the input text as C# code. This method doesn't return anything.
/// Callers may retrieve the result of this process by querying the FormattedCode property.
/// Passing null results in raising an exception.
/// </summary>
/// <param name="csharp">The text to be analyzed.</param>
protected void Parse(string csharp)
{
TokenClassifier classifier;
ConcreteToken csToken;
ClassificationTag tag;
Lexer lexer;
StringBuilder text;
int tokenIndex;
UiExceptionHelper.CheckNotNull(csharp, "csharp");
csharp = PreProcess(csharp);
lexer = new Lexer();
lexer.Parse(csharp);
classifier = new TokenClassifier();
text = new StringBuilder();
csToken = null;
tokenIndex = 0;
// loop through each token in the text
while (lexer.Next())
{
// classify the current token
tag = classifier.Classify(lexer.CurrentToken);
// if the tag cannot be merged with current csToken
// we flush csToken into _info and build a new instance
// from the current tag.
if (csToken == null ||
!csToken.CanMerge(_info.LineArray.Count, tag))
{
_flushToken(csToken, _info);
csToken = new ConcreteToken(
lexer.CurrentToken.Text, tag,
lexer.CurrentToken.IndexStart,
_info.LineArray.Count);
}
// append token's text into text
text.Append(lexer.CurrentToken.Text);
// handle newline character. Appends tokenIndex to LineArray
// and set tokenIndex to the start of the newline.
if (lexer.CurrentToken.Text == "\n")
{
_info.LineArray.Add(tokenIndex);
tokenIndex = _info.IndexArray.Count + 1;
}
}
// flush terminal token
_flushToken(csToken, _info);
if (csToken != null &&
_info.LineArray.Count == 0)
_info.LineArray.Add(tokenIndex);
_info.Text = csharp;
return;
}
/// <summary>
/// Analyzes the input text as C# code. This method doesn't return anything.
/// Callers may retrieve the result of this process by querying the FormattedCode property.
/// Passing null results in raising an exception.
/// </summary>
/// <param name="csharp">The text to be analyzed.</param>
protected void Parse(string csharp)
{
TokenClassifier classifier;
ConcreteToken csToken;
ClassificationTag tag;
Lexer lexer;
StringBuilder text;
int tokenIndex;
UiExceptionHelper.CheckNotNull(csharp, "csharp");
csharp = PreProcess(csharp);
lexer = new Lexer();
lexer.Parse(csharp);
classifier = new TokenClassifier();
text = new StringBuilder();
csToken = null;
tokenIndex = 0;
// loop through each token in the text
while (lexer.Next())
{
// classify the current token
tag = classifier.Classify(lexer.CurrentToken);
// if the tag cannot be merged with current csToken
// we flush csToken into _info and build a new instance
// from the current tag.
if (csToken == null ||
!csToken.CanMerge(_info.LineArray.Count, tag))
{
_flushToken(csToken, _info);
csToken = new ConcreteToken(
lexer.CurrentToken.Text, tag,
lexer.CurrentToken.IndexStart,
_info.LineArray.Count);
}
// append token's text into text
text.Append(lexer.CurrentToken.Text);
// handle newline character. Appends tokenIndex to LineArray
// and set tokenIndex to the start of the newline.
if (lexer.CurrentToken.Text == "\n")
{
_info.LineArray.Add(tokenIndex);
tokenIndex = _info.IndexArray.Count + 1;
}
}
// flush terminal token
_flushToken(csToken, _info);
if (csToken != null &&
_info.LineArray.Count == 0)
{
_info.LineArray.Add(tokenIndex);
}
_info.Text = csharp;
return;
}
