public IEnumerable <AstTokenizerBatchResult> AstTokenizerBatchResults()
{
InitializeNewBatch
(
minMemory: 0,
out IMemoryOwner <char> owner,
out Memory <char> newMemory,
out Memory <char> availableMemory,
out List <AstTokenizerBatchItem> currentResults
);
foreach (var rawLine in _lineReader.ReadLine())
{
var line = rawLine.ArraySegment;
AstTokenParserUtils.GetEssentialPart(line, out var lineDepth, out var essentialPart);
if (availableMemory.Length < essentialPart.Count)
{
if (currentResults.Count > 0)
{
yield return(new AstTokenizerBatchResult(currentResults, owner));
}
InitializeNewBatch(essentialPart.Count + 1, out owner, out newMemory, out availableMemory, out currentResults);
}
essentialPart.CopyTo(availableMemory.AsArraySegment());
var essentialCopy = availableMemory[..essentialPart.Count].AsReadOnlyArraySegment();
public List <AstToken> Parse(AstParserInterceptor interceptor)
{
var fastReader = new FastLineReader(_filePath);
var readerContext = new AstFileReaderContext(fastReader.ReadLine());
AstToken currentRoot = null;
var rootTokens = new List <AstToken>();
foreach (var rawLine in readerContext)
{
AstTokenParserUtils.GetEssentialPart(rawLine, out var lineDepth, out var line);
var token = new AstToken(true)
{
Line = readerContext.CurrentLine
};;
AstTokenParserUtils.ParseTokenDescription(token, line);
if (lineDepth == 0)
{
currentRoot = token;
rootTokens.Add(token);
continue;
}
if (currentRoot == null)
{
currentRoot = new AstToken(true)
{
unknownName = "Unknown"
};
rootTokens.Add(currentRoot);
}
if (interceptor.OnNodeParsed(readerContext, token, lineDepth))
{
continue;
}
currentRoot.AddChild(token, lineDepth - 1);
}
return(rootTokens);
}