public void Char()
{
string text = "aa 'x' bb '\\'' cc";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("aa", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("'x'", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("bb", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("'\\''", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("cc", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
text = "'\\t' '\\x21' '\\u0041' '\"' '\\\\' ";
scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("'\\t'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("'\\x21'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("'\\u0041'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("'\"'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual(@"'\\'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
}
public void Basics()
{
string text = @"using System.IO;
protected override Tuple2<NSMenu, int> GetScriptsLocation()
{
}
";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("using", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("System", scanner.Token.Text());
Assert.AreEqual(1, scanner.Token.Line);
scanner.Advance(); Assert.AreEqual(".", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("IO", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual(";", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("protected", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("override", scanner.Token.Text());
Assert.AreEqual(3, scanner.Token.Line);
scanner.Advance(); Assert.AreEqual("Tuple2", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("<", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("NSMenu", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual(",", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("int", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual(">", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("GetScriptsLocation", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("(", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual(")", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("{", scanner.Token.Text());
Assert.AreEqual(4, scanner.Token.Line);
scanner.Advance(); Assert.AreEqual("}", scanner.Token.Text());
Assert.AreEqual(5, scanner.Token.Line);
scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
}
public string ParseType(string text, int start, int stop, ref Token token)
{
Contract.Requires(text != null, "text is null");
Contract.Requires(start >= 0, "start is negative");
Contract.Requires(start <= text.Length, "start is too big");
Contract.Requires(start <= stop, "stop is too small");
var result = new StringBuilder();
try
{
m_text = text.Substring(start, stop - start);
m_scanner = new Scanner();
m_scanner.Init(m_text);
if (m_scanner.Token.Kind == TokenKind.Identifier)
{
DoParseType(result);
token = m_scanner.Token;
}
}
catch (LocalException)
{
result = null;
}
catch (ScannerException)
{
result = null;
}
catch (Exception e)
{
Log.WriteLine(TraceLevel.Warning, "LocalsParser", "Unexpected error:");
Log.WriteLine(TraceLevel.Warning, "LocalsParser", "{0}", e);
result = null;
}
return result != null && result.Length > 0 ? result.ToString() : null;
}
// This is an ad-hoc parser used with auto-complete. It would also be possible
// to write a real method body parser but that is an awful lot of work just to
// extract local declarations and is made trickier because the method body is
// being edited so it may be very malformed.
//
// What we do here is treat everything that could be a local variable as a
// local variable. This may result in a few false positives, but that should
// happen rarely and should not be a problem unless the false local hides
// a real one.
public Local[] Parse(string text, int start, int stop)
{
Contract.Requires(text != null, "text is null");
Contract.Requires(start >= 0, "start is negative");
Contract.Requires(start <= text.Length, "start is too big");
Contract.Requires(start <= stop, "stop is too small");
var locals = new List<Local>();
m_numErrors = 0;
try
{
m_text = text.Substring(start, stop - start);
m_scanner = new Scanner();
m_scanner.Init(m_text);
while (m_scanner.Token.IsValid())
{
if (m_scanner.Token.IsPunct("{"))
DoParseBlock(locals, 0);
else
m_scanner.Advance();
}
}
catch (ScannerException)
{
++m_numErrors;
}
return locals.ToArray();
}
public void VerbatimString()
{
string text = @"foo @""some
text
goes here"" yyy";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("foo", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("\"some\ntext\n goes here\"", scanner.Token.Text());
Assert.AreEqual(1, scanner.Token.Line);
scanner.Advance(); Assert.AreEqual("yyy", scanner.Token.Text());
Assert.AreEqual(3, scanner.Token.Line);
scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
text = "aa @\"bb\"\"cc\" dd";
scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("aa", scanner.Token.Text());
scanner.Advance();
Assert.AreEqual("\"bb\"\"cc\"", scanner.Token.Text());
scanner.Advance();
Assert.AreEqual("dd", scanner.Token.Text());
}
public void SingleLineComment()
{
string text = @"alpha // text goes here @"" hmm
some more // foo";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("alpha", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("some", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("more", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
}
public void String()
{
string text = "foo \"hmm\" xx \"aa { \" ";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("foo", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("\"hmm\"", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("xx", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("\"aa { \"", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
text = "xxx \"aa\\\"b\" yyy";
scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("xxx", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("\"aa\\\"b\"", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("yyy", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
}
public void Integers()
{
string text = "15 15U 0x1F 0x1FUL";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("15", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("15U", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("0x1F", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("0x1FUL", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
}
public void Preprocess()
{
string text = "alpha\n#region Your Name Here \n#define Foo\n#endif\n#endregion\n";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("alpha", scanner.Token.Text());
Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
CsPreprocess[] preprocess = scanner.Preprocess;
Assert.AreEqual(4, preprocess.Length);
Assert.AreEqual("region", preprocess[0].Name);
Assert.AreEqual("Your Name Here", preprocess[0].Text);
Assert.AreEqual("define", preprocess[1].Name);
Assert.AreEqual("Foo", preprocess[1].Text);
Assert.AreEqual("endif", preprocess[2].Name);
Assert.AreEqual("", preprocess[2].Text);
Assert.AreEqual("endregion", preprocess[3].Name);
Assert.AreEqual("", preprocess[3].Text);
}
public void Identifier()
{
string text = @"alpha _beta @gamma";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("alpha", scanner.Token.Text());
Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("_beta", scanner.Token.Text());
Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("gamma", scanner.Token.Text());
Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
}
public void IckyString()
{
string ick = @"#!/bin/sh
app=yyy
if [ ""$1"" = ""-?"" ] || [ ""$1"" = ""-h"" ] || [ ""$1"" = ""--help"" ] ; then
echo ""Usage: zzz [files]""
exit 0
fi
if [ -d ""$app"" ] ; then
open -a ""$app"" $@
else
echo ""Couldn't find $app: try re-installing the tool.""
fi
";
string text = string.Format("private const string Script = @\"{0}\"", ick.Replace("\"", "\"\""));
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("private", scanner.Token.Text());
scanner.Advance();
Assert.AreEqual("const", scanner.Token.Text());
scanner.Advance();
Assert.AreEqual("string", scanner.Token.Text());
scanner.Advance();
Assert.AreEqual("Script", scanner.Token.Text());
scanner.Advance();
Assert.AreEqual("=", scanner.Token.Text());
scanner.Advance();
Assert.AreEqual('"' + ick.Replace("\"", "\"\"") + '"', scanner.Token.Text());
scanner.Advance();
Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
}
public void Floats()
{
string text = "10.4 10.4M 10.4e100 .33 .33e-3 15f 15e100";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("10.4", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("10.4M", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("10.4e100", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual(".33", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual(".33e-3", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("15f", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual("15e100", scanner.Token.Text());
Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);
scanner.Advance();
Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
}
public void DelimitedComment()
{
string text = @"alpha /* this
is a comment */
foo";
var scanner = new Scanner();
scanner.Init(text);
Assert.AreEqual("alpha", scanner.Token.Text());
scanner.Advance(); Assert.AreEqual("foo", scanner.Token.Text());
Assert.AreEqual(3, scanner.Token.Line);
scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
}
private void DoInit(string text)
{
m_text = text;
m_scanner = new Scanner();
m_scanner.Init(text);
}
