public IEnumerable <AstTokenizerBatchResult> AstTokenizerBatchResults() { InitializeNewBatch ( minMemory: 0, out IMemoryOwner <char> owner, out Memory <char> newMemory, out Memory <char> availableMemory, out List <AstTokenizerBatchItem> currentResults ); foreach (var rawLine in _lineReader.ReadLine()) { var line = rawLine.ArraySegment; AstTokenParserUtils.GetEssentialPart(line, out var lineDepth, out var essentialPart); if (availableMemory.Length < essentialPart.Count) { if (currentResults.Count > 0) { yield return(new AstTokenizerBatchResult(currentResults, owner)); } InitializeNewBatch(essentialPart.Count + 1, out owner, out newMemory, out availableMemory, out currentResults); } essentialPart.CopyTo(availableMemory.AsArraySegment()); var essentialCopy = availableMemory[..essentialPart.Count].AsReadOnlyArraySegment();
public void SkipSubTree() { var enumerator = _enumerator; var origLineDepth = AstTokenParserUtils.GetLineDepth(enumerator.Current.Span); var tempDepth = origLineDepth + 1; while (origLineDepth < tempDepth && enumerator.MoveNext()) { CurrentLine++; tempDepth = AstTokenParserUtils.GetLineDepth(enumerator.Current.Span); } _shouldMoveNext = false; }
public List <AstToken> Parse(AstParserInterceptor interceptor) { var fastReader = new FastLineReader(_filePath); var readerContext = new AstFileReaderContext(fastReader.ReadLine()); AstToken currentRoot = null; var rootTokens = new List <AstToken>(); foreach (var rawLine in readerContext) { AstTokenParserUtils.GetEssentialPart(rawLine, out var lineDepth, out var line); var token = new AstToken(true) { Line = readerContext.CurrentLine };; AstTokenParserUtils.ParseTokenDescription(token, line); if (lineDepth == 0) { currentRoot = token; rootTokens.Add(token); continue; } if (currentRoot == null) { currentRoot = new AstToken(true) { unknownName = "Unknown" }; rootTokens.Add(currentRoot); } if (interceptor.OnNodeParsed(readerContext, token, lineDepth)) { continue; } currentRoot.AddChild(token, lineDepth - 1); } return(rootTokens); }
public IEnumerable <AstTokenizerResult> TokenizeLines() { //ResizeBuffer(10 * 1024, null, out var bufferOwner, out var buffer); foreach (var rawLine in _lineReader.ReadLine()) { var line = rawLine.ArraySegment; var lineDepth = 0; var tokenStart = -1; var tokenEnd = -1; for (var i = 0; i < line.Count; i += 2) { if (line[i] == '|' || line[i] == '`' || line[i] == ' ' || line[i] == '-') { lineDepth++; continue; } tokenStart = i; break; } //var required = line.Span[tokenStart..]; //if (required.Length > buffer.Length) //{ // ResizeBuffer(required.Length, null, out bufferOwner, out buffer); //} //required.CopyTo(buffer.Span); // This could be done later at any other thread tho... AstTokenParserUtils.ParseTokenAndDescription(line, tokenStart, out var token, out var declaration); yield return(new AstTokenizerResult(lineDepth, token, declaration)); //yield return new AstTokenizerResult(lineDepth, ReadOnlyArraySegment<char>.Empty, ReadOnlyArraySegment<char>.Empty); } }