/// <summary>
/// Представить исходный код, содержащийся в файле, в виде набора токенов
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
public override CCodeUnitsCollection Tokenize(CTokenizerParms args)
{
CCodeUnitsCollection result_collection = new CCodeUnitsCollection();
Scanner lex = new Scanner(args.GetPath());
CTokensIterator iter = new CTokensIterator(lex);
foreach (Token token in iter)
{
result_collection.Add(ExtCodeUnitCreator.FromToken(token, args));
}
return(result_collection);
}
private void BuildCCodeUnitsCollection(MethodBlock mb)
{
code_units_collection = new CCodeUnitsCollection();
foreach (Token token in iter)
{
if (IsTokenBelongMethodBlock(token, mb))
{
code_units_collection.Add(ExtCodeUnitCreator.FromToken(token, m_args));
}
else
{
if (IsTokenAfterMethodBlock(token, mb))
{
const string METHOD_END_TOKEN = "}";
if ((token.val == METHOD_END_TOKEN) && (code_units_collection.back().Text != METHOD_END_TOKEN))
{
code_units_collection.Add(ExtCodeUnitCreator.FromToken(token, m_args));
}
break;
}
}
}
}