public static bool EndContext(Input input, int depth = 2) { if (Char2.IsNewLine(input.Current())) { var index = 0; while (input.HasPeek(index) && input.Peek(index) == '↓') { index++; } var contextEnded = index == depth && input.HasPeek(index) && input.Peek(index) != '→'; if (contextEnded) { for (var i = 0; i < index; ++i) { input.Next(); } } return(contextEnded); } else { return(false); } }
public static Token TakeLine(Input input, TokenType type = TokenType.Other) { var start = input.Position; var startColumn = input.Column; var startLine = input.Line; StringBuilder builder = new StringBuilder(); builder.Append(input.Current()); while (input.HasNext() && !Char2.IsNewLine(input.Next())) { builder.Append(input.Current()); } return(new Token() { StartIndex = start, StartColumn = startColumn, StartLine = startLine, EndIndex = input.Position, EndColumn = input.Column, EndLine = input.Line, Value = builder.ToString(), TokenType = type }); }
public IEnumerable <Token> Lex(string code) { var preparedSource = PrepareSource(code); var input = new Input(preparedSource); var context = false; while (input.HasNext()) { if (input.IsEqualTo("type") || input.IsEqualTo("alias") || input.IsEqualTo("data") || input.IsEqualTo("choice") || input.IsEqualTo("open") || input.IsEqualTo("view") || input.IsEqualTo("flow") || input.IsEqualTo("aggregate") || input.IsEqualTo("entity")) { context = true; yield return(new Token() { TokenType = TokenType.ContextStarted }); yield return(TokenLexers.Word(input)); } else if (input.IsEqualTo("extends") || input.IsEqualTo("importing") || input.IsEqualTo("compose") || input.IsEqualTo("loop") || input.IsEqualTo("pluck")) { yield return(TokenLexers.Word(input)); } else if (input.IsEqualTo("->")) { yield return(TokenLexers.Operators(input, "->")); } else if (input.IsEqualTo("::")) { yield return(TokenLexers.Operators(input, "::")); } else if (!context && Char.IsLetter(input.Current())) { yield return(TokenLexers.TakeUntillEndOfContext(input)); yield return(new Token() { TokenType = TokenType.ContextEnded }); } else if (context && Char2.IsNewLine(input.Current()) && TokenLexers.EndContext(input, 1)) { context = false; yield return(new Token() { TokenType = TokenType.ContextEnded }); } else if (context && Char.IsNumber(input.Current())) { yield return(TokenLexers.Number(input)); } else if (context && Char.IsUpper(input.Current())) { yield return(TokenLexers.Identifier(input)); } else if (context && input.Current() == '\'') { yield return(TokenLexers.GenericParameter(input)); } else if (context && Char.IsLower(input.Current())) { yield return(TokenLexers.Word(input)); } else if (context && input.Current() == '&') { yield return(TokenLexers.Take(input, TokenType.And)); } else if (context && input.Current() == '/') { yield return(TokenLexers.Pattern(input)); } else if (context && input.Current() == '"') { yield return(TokenLexers.String(input)); } else if (context && char.IsNumber(input.Current())) { yield return(TokenLexers.Number(input)); } else if (input.Current() == '{') { yield return(TokenLexers.Comment(input)); } else if (input.Current() == '#') { yield return(TokenLexers.Chapter(input)); } else if (input.Current() == '@') { yield return(TokenLexers.Annotation(input)); } else if (input.Current() == '%') { yield return(TokenLexers.Directive(input)); } else if (input.Current() == ';') { yield return(TokenLexers.Take(input, TokenType.EndStatement)); } else if (Char2.Indent(input.Current())) { yield return(TokenLexers.Take(input, TokenType.Indent)); } else if (Char2.Equal(input.Current())) { yield return(TokenLexers.Take(input, TokenType.Equal)); } else if (Char2.Or(input.Current())) { yield return(TokenLexers.Take(input, TokenType.Or)); } else if (Char2.Separator(input.Current())) { yield return(TokenLexers.Take(input, TokenType.Separator)); } else if (input.Current() == '(') { yield return(TokenLexers.Take(input, TokenType.GroupOpen)); } else if (input.Current() == ')') { yield return(TokenLexers.Take(input, TokenType.GroupClosed)); } else if (input.Current() == ',') { yield return(TokenLexers.Take(input, TokenType.ListSeparator)); } else if (Char2.IsNewLine(input.Current())) { var newline = TokenLexers.Take(input, TokenType.NewLine); newline.Value = "↓"; yield return(newline); } else if (Char.IsWhiteSpace(input.Current())) { var whiteSpace = TokenLexers.Whitespace(input); if (!_ignoreWhiteSpace) { yield return(whiteSpace); } } else if (!context) { yield return(TokenLexers.TakeUntillEndOfContext(input)); } else { input.Next(); } } // If we are still inside of a context when the code has been parsed, // we'll want to end the context, jsut for good measure! if (context) { yield return(new Token() { TokenType = TokenType.ContextEnded }); context = false; } yield return(new Token() { TokenType = TokenType.EndOfFile }); }