[Test] public void QuoteDelimited()
{
StructLexer lexer = new StructLexer("struct \"My Structure\" { }");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.AreEqual("My Structure", lexer.GetNextToken(StructTokenType.String));
}
private static void LoadAttributes(StructLexer lexer, List<Attribute> attrs)
{
if (lexer.PeekNextToken() == StructTokenType.OpenSquare)
{
lexer.GetNextToken(StructTokenType.OpenSquare);
while(true)
{
TextPosition pos = lexer.CurrentPosition;
string attrName = lexer.GetNextToken(StructTokenType.String);
string attrValue;
if (lexer.CheckNextToken(StructTokenType.Equals))
{
attrValue = lexer.GetAttributeValue(out pos);
}
else
{
attrValue = "1";
}
attrs.Add(new Attribute(attrName, attrValue, pos));
if (lexer.CheckNextToken(StructTokenType.CloseSquare)) break;
if (!lexer.CheckNextToken(StructTokenType.Comma))
{
throw new Exception("Unexpected token");
}
}
}
}
[Test] public void ParenthesesDelimited()
{
StructLexer lexer = new StructLexer("struct (My (Best) Structure) { }");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.AreEqual("My (Best) Structure", lexer.GetNextToken(StructTokenType.String));
}
public void CStyleComment()
{
StructLexer lexer = new StructLexer("/* test \nalso test\nend*/ struct A { }");
Assert.AreEqual(StructTokenType.String, lexer.PeekNextToken());
Assert.AreEqual(3, lexer.CurrentPosition.Line);
Assert.AreEqual("struct", lexer.GetNextToken(StructTokenType.String));
}
[Test] public void CStyleComment()
{
StructLexer lexer = new StructLexer("/* test \nalso test\nend*/ struct A { }");
Assert.AreEqual(StructTokenType.String, lexer.PeekNextToken());
Assert.AreEqual(3, lexer.CurrentPosition.Line);
Assert.AreEqual("struct", lexer.GetNextToken(StructTokenType.String));
}
public void LineBreak()
{
StructLexer lexer = new StructLexer("struct BITMAPINFOHEADER {\n}\n");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.CloseCurly));
Assert.IsTrue(lexer.EndOfStream());
}
public void LineAndCol()
{
StructLexer lexer = new StructLexer("struct A {\r\n u8");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
TextPosition pos = lexer.CurrentPosition;
Assert.AreEqual(2, pos.Line);
Assert.AreEqual(3, pos.Col);
}
[Test] public void LineBreak()
{
StructLexer lexer = new StructLexer("struct BITMAPINFOHEADER {\n}\n");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.CloseCurly));
Assert.IsTrue(lexer.EndOfStream());
}
public void MultilineAttributeValue()
{
var lexer = new StructLexer("[FileName=(a\nb)] struct A { }");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenSquare));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.Equals));
TextPosition pos;
var value = lexer.GetAttributeValue(out pos);
Assert.AreEqual("(a\nb)", value);
Assert.AreEqual(2, lexer.CurrentPosition.Line);
}
public void MultilineDelimitedString()
{
var lexer = new StructLexer("struct A { if (\na)");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
var token = lexer.GetNextToken(StructTokenType.String);
Assert.AreEqual("\na", token);
Assert.AreEqual(2, lexer.CurrentPosition.Line);
}
[Test] public void LineAndCol()
{
StructLexer lexer = new StructLexer("struct A {\r\n u8");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
TextPosition pos = lexer.CurrentPosition;
Assert.AreEqual(2, pos.Line);
Assert.AreEqual(3, pos.Col);
}
[Test] public void SimpleLexer()
{
StructLexer lexer = new StructLexer("struct BITMAPINFOHEADER { }");
Assert.AreEqual(StructTokenType.String, lexer.PeekNextToken());
Assert.AreEqual("struct", lexer.GetNextToken(StructTokenType.String));
Assert.AreEqual(StructTokenType.String, lexer.PeekNextToken());
Assert.AreEqual("BITMAPINFOHEADER", lexer.GetNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.CloseCurly));
Assert.IsTrue(lexer.EndOfStream());
}
[Test] public void MultilineAttributeValue()
{
var lexer = new StructLexer("[FileName=(a\nb)] struct A { }");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenSquare));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.Equals));
TextPosition pos;
var value = lexer.GetAttributeValue(out pos);
Assert.AreEqual("(a\nb)", value);
Assert.AreEqual(2, lexer.CurrentPosition.Line);
}
[Test] public void MultilineDelimitedString()
{
var lexer = new StructLexer("struct A { if (\na)");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
var token = lexer.GetNextToken(StructTokenType.String);
Assert.AreEqual("\na", token);
Assert.AreEqual(2, lexer.CurrentPosition.Line);
}
[Test] public void ParseException()
{
ParseException exception = null;
try
{
StructLexer lexer = new StructLexer("struct A {\r\n !");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
lexer.PeekNextToken();
}
catch (ParseException e)
{
exception = e;
}
Assert.IsNotNull(exception);
Assert.AreEqual(2, exception.Position.Line);
Assert.AreEqual(3, exception.Position.Col);
}
public void ParseException()
{
ParseException exception = null;
try
{
StructLexer lexer = new StructLexer("struct A {\r\n !");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
lexer.PeekNextToken();
}
catch(ParseException e)
{
exception = e;
}
Assert.IsNotNull(exception);
Assert.AreEqual(2, exception.Position.Line);
Assert.AreEqual(3, exception.Position.Col);
}
public void UnclosedParen()
{
StructLexer lexer = new StructLexer("(abc");
lexer.PeekNextToken();
}
private void LoadFieldGroup(StructLexer lexer, StructDef structDef, StructField parentField)
{
lexer.GetNextToken(StructTokenType.OpenCurly);
StructField linkToField = null;
while(lexer.PeekNextToken() != StructTokenType.CloseCurly)
{
StructField field = LoadField(lexer, structDef, parentField);
if (field != null)
{
bool isLinked = false;
if (linkToField != null)
isLinked = linkToField.CanLinkField(field);
if (isLinked)
linkToField.LinkField(field);
else
linkToField = field;
field.Validate();
}
}
lexer.GetNextToken(StructTokenType.CloseCurly);
}
private void LoadPlugin(StructLexer lexer)
{
string pluginFile = lexer.GetNextToken(StructTokenType.String);
_curStructFile.AddPlugin(pluginFile);
lexer.GetNextToken(StructTokenType.Semicolon);
}
[Test] public void EndOfLineComments()
{
StructLexer lexer = new StructLexer("// comment\nstruct BITMAPINFOHEADER {\n}\n");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
}
private void LoadStruct(StructLexer lexer, List<Attribute> attrs)
{
string name = lexer.GetNextToken(StructTokenType.String);
if (_curStructFile.GetStructByName(name) != null)
throw new Exception("Duplicate structure name '" + name + "'");
LoadAttributes(lexer, attrs);
StructDef structDef = new StructDef(_curStructFile, name);
foreach (Attribute attr in attrs)
{
try
{
structDef.SetAttribute(attr.Key, attr.Value, attr.Position);
}
catch(ParseException ex)
{
_errors.Add(ex);
}
}
LoadFieldGroup(lexer, structDef, null);
_curStructFile.Add(structDef);
}
public void UnclosedParen()
{
StructLexer lexer = new StructLexer("(abc");
lexer.PeekNextToken();
}
public void QuoteDelimited()
{
StructLexer lexer = new StructLexer("struct \"My Structure\" { }");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.AreEqual("My Structure", lexer.GetNextToken(StructTokenType.String));
}
public void ParenthesesDelimited()
{
StructLexer lexer = new StructLexer("struct (My (Best) Structure) { }");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
Assert.AreEqual("My (Best) Structure", lexer.GetNextToken(StructTokenType.String));
}
public void EndOfLineComments()
{
StructLexer lexer = new StructLexer("// comment\nstruct BITMAPINFOHEADER {\n}\n");
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.String));
}
private StructField LoadField(StructLexer lexer, StructDef structDef, StructField parentField)
{
List<Attribute> attrs = new List<Attribute>();
LoadAttributes(lexer, attrs);
TextPosition fieldPosition = lexer.CurrentPosition;
string fieldType = lexer.GetNextToken(StructTokenType.String);
StructField field = null;
try
{
field = _fieldFactory.CreateField(structDef, fieldType, _attributeRegistry);
field.Name = fieldType;
field.Position = fieldPosition;
}
catch(Exception ex)
{
_errors.Add(new ParseException(ex.Message, fieldPosition));
}
LoadAttributes(lexer, attrs);
if (lexer.PeekNextToken() != StructTokenType.Semicolon && lexer.PeekNextToken() != StructTokenType.OpenCurly)
{
TextPosition pos = lexer.CurrentPosition;
string tag = lexer.GetNextToken(StructTokenType.String);
LoadAttributes(lexer, attrs);
if (field != null)
_attributeRegistry.SetFieldAttribute(field, field.DefaultAttribute, tag, pos);
}
foreach (Attribute attr in attrs)
_attributeRegistry.SetFieldAttribute(field, attr.Key, attr.Value, attr.Position);
if (lexer.PeekNextToken() == StructTokenType.OpenCurly)
LoadFieldGroup(lexer, structDef, field);
else
lexer.GetNextToken(StructTokenType.Semicolon);
if (field != null)
{
field.EndPosition = lexer.LastTokenEndPosition;
if (parentField == null)
structDef.AddField(field);
else
parentField.AddChildField(field);
}
return field;
}
private void LoadStructFile(string fileName, StructSourceContext context)
{
StructLexer lexer = new StructLexer(fileName, context.GetSourceText(fileName));
while(!lexer.EndOfStream())
{
List<Attribute> attrs = new List<Attribute>();
LoadAttributes(lexer, attrs);
string token = lexer.GetNextToken(StructTokenType.String);
if (token == "struct")
LoadStruct(lexer, attrs);
else if (token == "enum")
LoadEnum(lexer, attrs);
else if (token == "alias")
LoadAlias(lexer, attrs);
else if (token == "plugin")
LoadPlugin(lexer);
else if (token == "include")
{
string includeName = lexer.GetNextToken(StructTokenType.String);
lexer.GetNextToken(StructTokenType.Semicolon);
LoadStructFile(includeName, context);
}
else
throw new Exception("Unexpected top-level item " + token);
}
}
private void LoadAlias(StructLexer lexer, List<Attribute> attrs)
{
string baseName = lexer.GetNextToken(StructTokenType.String);
LoadAttributes(lexer, attrs);
string aliasName = lexer.GetNextToken(StructTokenType.String);
LoadAttributes(lexer, attrs);
lexer.GetNextToken(StructTokenType.Semicolon);
_fieldFactory.RegisterAlias(aliasName, baseName, attrs);
}
private void LoadEnum(StructLexer lexer, List<Attribute> attrs)
{
string name = lexer.GetNextToken(StructTokenType.String);
EnumDef enumDef = new EnumDef(_curStructFile, name);
LoadAttributes(lexer, attrs);
foreach (Attribute attr in attrs)
enumDef.SetAttribute(attr.Key, attr.Value);
lexer.GetNextToken(StructTokenType.OpenCurly);
uint lastValue = UInt32.MaxValue;
while(!lexer.CheckNextToken(StructTokenType.CloseCurly))
{
string constName = lexer.GetNextToken(StructTokenType.String);
if (lexer.CheckNextToken(StructTokenType.Equals))
{
string constValue = lexer.GetNextToken(StructTokenType.String);
lastValue = (uint) ExpressionParser.Parse(constValue).EvaluateInt(null);
}
else
lastValue++;
enumDef.AddValue(constName, lastValue);
if (!lexer.CheckNextToken(StructTokenType.Comma))
{
if (lexer.PeekNextToken() != StructTokenType.CloseCurly)
throw new Exception("Unexpected token in enum: " + lexer.PeekNextToken());
}
}
_curStructFile.Add(enumDef);
}
public void SimpleLexer()
{
StructLexer lexer = new StructLexer("struct BITMAPINFOHEADER { }");
Assert.AreEqual(StructTokenType.String, lexer.PeekNextToken());
Assert.AreEqual("struct", lexer.GetNextToken(StructTokenType.String));
Assert.AreEqual(StructTokenType.String, lexer.PeekNextToken());
Assert.AreEqual("BITMAPINFOHEADER", lexer.GetNextToken(StructTokenType.String));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.OpenCurly));
Assert.IsTrue(lexer.CheckNextToken(StructTokenType.CloseCurly));
Assert.IsTrue(lexer.EndOfStream());
}
