private IEnumerable <BatchLineReadResult> EnumerateBatches()
{
IMemoryOwner <char> memoryOwner = null;
Memory <char> memory = default;
var availableMemory = Memory <char> .Empty;
List <BatchLineItem> currentResults = default;
foreach (var rawLine in _lineReader.ReadLine())
{
if (availableMemory.Length < rawLine.Count)
{
if (currentResults?.Count > 0)
{
yield return(new BatchLineReadResult
{
Items = currentResults,
Disposable = memoryOwner
});
}
InitializeNewBatch(rawLine.Count + 1, out memoryOwner, out memory, out availableMemory, out currentResults);
}
rawLine.CopyTo(availableMemory.Span);
var lineCopy = availableMemory[..rawLine.Count];
public IEnumerable <AstTokenizerResult> TokenizeLines()
{
//ResizeBuffer(10 * 1024, null, out var bufferOwner, out var buffer);
foreach (var rawLine in _lineReader.ReadLine())
{
var line = rawLine.ArraySegment;
var lineDepth = 0;
var tokenStart = -1;
var tokenEnd = -1;
for (var i = 0; i < line.Count; i += 2)
{
if (line[i] == '|' || line[i] == '`' || line[i] == ' ' || line[i] == '-')
{
lineDepth++;
continue;
}
tokenStart = i;
break;
}
//var required = line.Span[tokenStart..];
//if (required.Length > buffer.Length)
//{
// ResizeBuffer(required.Length, null, out bufferOwner, out buffer);
//}
//required.CopyTo(buffer.Span);
// This could be done later at any other thread tho...
AstTokenParserUtils.ParseTokenAndDescription(line, tokenStart, out var token, out var declaration);
yield return(new AstTokenizerResult(lineDepth, token, declaration));
//yield return new AstTokenizerResult(lineDepth, ReadOnlyArraySegment<char>.Empty, ReadOnlyArraySegment<char>.Empty);
}
}
public List <AstToken> Parse(AstParserInterceptor interceptor)
{
var fastReader = new FastLineReader(_filePath);
var readerContext = new AstFileReaderContext(fastReader.ReadLine());
AstToken currentRoot = null;
var rootTokens = new List <AstToken>();
foreach (var rawLine in readerContext)
{
AstTokenParserUtils.GetEssentialPart(rawLine, out var lineDepth, out var line);
var token = new AstToken(true)
{
Line = readerContext.CurrentLine
};;
AstTokenParserUtils.ParseTokenDescription(token, line);
if (lineDepth == 0)
{
currentRoot = token;
rootTokens.Add(token);
continue;
}
if (currentRoot == null)
{
currentRoot = new AstToken(true)
{
unknownName = "Unknown"
};
rootTokens.Add(currentRoot);
}
if (interceptor.OnNodeParsed(readerContext, token, lineDepth))
{
continue;
}
currentRoot.AddChild(token, lineDepth - 1);
}
return(rootTokens);
}