public List <IToken> IntGetTokens(string sourceText, IFileSource fileSource, OffsetsStore offsetsStore)
{
var clearText = ClearSourceText(sourceText, offsetsStore);
SearchFsa <char> fsa = PrepareFsa();
var tokens = new List <IToken>();
var ep = fsa.GetEndPoint((index, context) =>
{
var currentWord = context.ToString();
var token = new CsToken()
{
Code = CSharpKeywords.Keywords[currentWord],
Position = index,
FileSource = fileSource,
StringLength = currentWord.Length
};
tokens.Add(token);
});
foreach (var ch in sourceText)
{
ep.ProcessIncome(ch);
}
return(tokens);
}
public string ClearSourceText(string sourceText, OffsetsStore offsetsStore)
{
var tree = ParseText(sourceText);
var root = tree.GetRoot();
var rewriter = new Rewriter(offsetsStore);
var result = rewriter.Visit(root);
return(result.ToFullString().ToLowerInvariant());
}
public Rewriter(OffsetsStore offsetsStore)
{
this.Offsets = offsetsStore;
}
public List <IToken> GetTokens(IFileSource fileSource, OffsetsStore offsetsStore)
{
offsetsStore = offsetsStore ?? new OffsetsStore();
return(IntGetTokens(fileSource.GetData(), fileSource, offsetsStore));
}