public List<TokenDefinition> Parse()
{
List<TokenDefinition> definitions = new List<TokenDefinition>();
if (!Valid) return definitions;
StreamReader rd = new StreamReader(Filename);
while (rd.EndOfStream == false)
{
string line = rd.ReadLine();
if (!string.IsNullOrEmpty(line))
{
TokenDefinition token = null;
if(line.StartsWith("//"))
{
// skip annotations
continue;
}
if (line.Contains(' '))
{
var value = line.Split(' ');
bool ignored = false;
var attributes = new TokenAttributes();
if (value.Length > 2)
{
for (var i = 2; i < value.Length; i++)
{
if (value[i].ToLower().Contains("ignore"))
ignored = true;
else
{
var _value = value[i].ToString();
if (string.IsNullOrEmpty(_value) == false)
attributes.Add(value[i].ToString());
}
}
}
token = new TokenDefinition(value[0], new Regex(value[1]), ignored);
token.Attributes = attributes;
}
else
{
token = new TokenDefinition(line, new Regex(line));
}
if (token != null)
definitions.Add(token);
}
}
return definitions;
}
public static Token Parse(string line)
{
var pieces = parsePieces(line);
if (pieces.Count <= 4)
return null;
try
{
var position = new TokenPosition(int.Parse(pieces[2]), int.Parse(pieces[3]), int.Parse(pieces[4]));
var attributes = new TokenAttributes();
if(pieces.Count >= 5)
{
for (int i = 5; i < pieces.Count; i++)
attributes.Add(pieces[i]);
}
var token = new Token(pieces[0], pieces[1], position, attributes);
return token;
}
catch (Exception)
{
return null;
}
}