public Span Merge(Span other) { if (this.unit == null) return other; if (other.unit == null) return this; if (this.unit != other.unit) throw new InvalidOperationException("attempted to merge spans from different units"); return new Span( this.unit, Math.Min(this.start, other.start), Math.Max(this.end, other.end)); }
public static TokenCollection Tokenize(Core.Session session, Core.TextInput input) { var output = new TokenCollection(); // Iterate through all characters in input. var index = 0; while (index < input.Length()) { // Skip whitespace. if (IsWhitespace(input[index])) { index++; continue; } // Match next characters to a token, and add it to the output. var match = TryMatchModelToken(input, index) ?? TryMatchVaryingToken(input, index) ?? new TokenMatch(new string(input[index], 1), TokenKind.Error); var span = new Span(input, index, index + match.representation.Length); // Signal errors. if (match.kind == TokenKind.Error) { session.AddMessage(MessageKind.Error, MessageCode.UnexpectedChar, "unexpected character", span); } // Skip line comments. else if (match.kind == TokenKind.DoubleSlash) { while (index < input.Length() && input[index] != '\n') index++; continue; } // Skip multiline comments. else if (match.kind == TokenKind.SlashAsterisk) { var nesting = 1; index += 2; while (index < input.Length() - 1) { if (input[index] == '*' && input[index + 1] == '/') { nesting--; index += 2; if (nesting == 0) break; } else if (input[index] == '/' && input[index + 1] == '*') { nesting++; index += 2; } else index++; } if (nesting > 0) session.AddMessage(MessageKind.Error, MessageCode.UnexpectedChar, "unterminated comment", span); continue; } output.tokens.Add(new Token(match.kind, span)); index += match.representation.Length; } output.tokenAfterEnd = new Token(TokenKind.Error, new Diagnostics.Span(input, index, index)); return output; }