public IEnumerable <AstTokenizerBatchResult> AstTokenizerBatchResults()
{
InitializeNewBatch
(
minMemory: 0,
out IMemoryOwner <char> owner,
out Memory <char> newMemory,
out Memory <char> availableMemory,
out List <AstTokenizerBatchItem> currentResults
);
foreach (var rawLine in _lineReader.ReadLine())
{
var line = rawLine.ArraySegment;
AstTokenParserUtils.GetEssentialPart(line, out var lineDepth, out var essentialPart);
if (availableMemory.Length < essentialPart.Count)
{
if (currentResults.Count > 0)
{
yield return(new AstTokenizerBatchResult(currentResults, owner));
}
InitializeNewBatch(essentialPart.Count + 1, out owner, out newMemory, out availableMemory, out currentResults);
}
essentialPart.CopyTo(availableMemory.AsArraySegment());
var essentialCopy = availableMemory[..essentialPart.Count].AsReadOnlyArraySegment();
public void SkipSubTree()
{
var enumerator = _enumerator;
var origLineDepth = AstTokenParserUtils.GetLineDepth(enumerator.Current.Span);
var tempDepth = origLineDepth + 1;
while (origLineDepth < tempDepth && enumerator.MoveNext())
{
CurrentLine++;
tempDepth = AstTokenParserUtils.GetLineDepth(enumerator.Current.Span);
}
_shouldMoveNext = false;
}
public List <AstToken> Parse(AstParserInterceptor interceptor)
{
var fastReader = new FastLineReader(_filePath);
var readerContext = new AstFileReaderContext(fastReader.ReadLine());
AstToken currentRoot = null;
var rootTokens = new List <AstToken>();
foreach (var rawLine in readerContext)
{
AstTokenParserUtils.GetEssentialPart(rawLine, out var lineDepth, out var line);
var token = new AstToken(true)
{
Line = readerContext.CurrentLine
};;
AstTokenParserUtils.ParseTokenDescription(token, line);
if (lineDepth == 0)
{
currentRoot = token;
rootTokens.Add(token);
continue;
}
if (currentRoot == null)
{
currentRoot = new AstToken(true)
{
unknownName = "Unknown"
};
rootTokens.Add(currentRoot);
}
if (interceptor.OnNodeParsed(readerContext, token, lineDepth))
{
continue;
}
currentRoot.AddChild(token, lineDepth - 1);
}
return(rootTokens);
}
public IEnumerable <AstTokenizerResult> TokenizeLines()
{
//ResizeBuffer(10 * 1024, null, out var bufferOwner, out var buffer);
foreach (var rawLine in _lineReader.ReadLine())
{
var line = rawLine.ArraySegment;
var lineDepth = 0;
var tokenStart = -1;
var tokenEnd = -1;
for (var i = 0; i < line.Count; i += 2)
{
if (line[i] == '|' || line[i] == '`' || line[i] == ' ' || line[i] == '-')
{
lineDepth++;
continue;
}
tokenStart = i;
break;
}
//var required = line.Span[tokenStart..];
//if (required.Length > buffer.Length)
//{
// ResizeBuffer(required.Length, null, out bufferOwner, out buffer);
//}
//required.CopyTo(buffer.Span);
// This could be done later at any other thread tho...
AstTokenParserUtils.ParseTokenAndDescription(line, tokenStart, out var token, out var declaration);
yield return(new AstTokenizerResult(lineDepth, token, declaration));
//yield return new AstTokenizerResult(lineDepth, ReadOnlyArraySegment<char>.Empty, ReadOnlyArraySegment<char>.Empty);
}
}
