public Token TryLex(GrammarDefinition grammar, LineColumnMapping mapping, string source, ref int offset)
{
for (int i = 0; i < Keyword.Length; i++)
{
if ((offset + i) >= source.Length)
{
return(null);
}
if (grammar.IsCaseInsensitive)
{
if (char.ToLowerInvariant(source[offset + i]) != char.ToLowerInvariant(Keyword[i]) &&
char.ToUpperInvariant(source[offset + i]) != char.ToUpperInvariant(Keyword[i]))
{
return(null);
}
}
else
{
if (source[offset + i] != Keyword[i])
{
return(null);
}
}
}
var original = offset;
offset += Keyword.Length;
return(new Token(Keyword, source.Substring(original, Keyword.Length), mapping.GetLocation(original)));
}
public Token TryLex(GrammarDefinition grammar, LineColumnMapping mapping, string source, ref int offset)
{
int original = offset;
if (Regex.Regex.Lex(grammar, source, ref offset))
{
return(new Token(Regex.Name, source.Substring(original, offset - original), mapping.GetLocation(original)));
}
else
{
return(null);
}
}
public static Token[] Lex(string source)
{
List <Token> result = new List <Token>();
var mapping = new LineColumnMapping(source);
ILexer[] lexers = new ILexer[]
{
new CommentLexer(mapping, source),
new EofLexer(mapping, source),
new EscapeSequenceLexer(mapping, source),
new IdentifierLexer(mapping, source),
new OperatorLexer(mapping, source),
new StringLiteralLexer(mapping, source),
new WhitespaceLexer(mapping, source)
};
int offset = 0;
Token lastToken = null;
do
{
for (int i = 0; i < lexers.Length; i++)
{
lastToken = lexers[i].TryLex(ref offset);
if (lastToken != null)
{
break;
}
}
if (lastToken == null)
{
throw new GrammarException(mapping.GetLocation(offset), $"caractere inesperado {source[offset]}");
}
result.Add(lastToken);
}while (lastToken.Type != TokenType.Eof);
return(result.Where(x => x.Type != TokenType.Whitespace && x.Type != TokenType.Comment).ToArray());
}
public static IEnumerable <Token> Lex(GrammarDefinition definition, string source)
{
var lexers = GetLexers(definition);
var bySize = new List <Token>();
int offset = 0;
var mapping = new LineColumnMapping(source);
while (offset < source.Length)
{
bySize.Clear();
var original = offset;
foreach (var item in lexers)
{
var tk = item.TryLex(definition, mapping, source, ref offset);
if (tk != null)
{
offset = original;
bySize.Add(tk);
}
}
if (bySize.Count == 0)
{
throw new GrammarException(mapping.GetLocation(offset), $"erro do analisador léxico próximo de '{source[offset]}'");
}
else
{
var maxSize = bySize.Max(x => x.Source.Length);
var tk = bySize.Where(x => x.Source.Length == maxSize).First();
if (tk.Name != Token.WhitespaceName)
{
yield return(tk);
}
offset += tk.Source.Length;
}
}
yield return(new Token(Token.EofName, string.Empty, mapping.GetLocation(source.Length)));
}
public EscapeSequenceLexer(LineColumnMapping mapping, string source) : base(mapping, source)
{
}
public BaseLexer(LineColumnMapping mapping, string source)
{
_Mapping = mapping;
_Source = source;
}
public Token TryLex(GrammarDefinition grammar, LineColumnMapping mapping, string source, ref int offset)
{
if (char.IsWhiteSpace(source[offset]))
{
int original = offset;
while (offset < source.Length && char.IsWhiteSpace(source[offset]))
{
offset++;
}
return(new Token(Token.WhitespaceName, source.Substring(original, offset - original), mapping.GetLocation(original)));
}
else
{
Token line_tk = null;
if (!string.IsNullOrEmpty(grammar.LineComment))
{
line_tk = TryLexKeyword(grammar.LineComment, grammar, mapping, source, ref offset);
}
if (line_tk != null)
{
int original = offset;
while (offset < source.Length && source[offset] != '\n')
{
offset++;
}
return(new Token(Token.WhitespaceName, line_tk.Source + source.Substring(original, offset - original), line_tk.Location));
}
else
{
var before_block_tk = offset;
Token block_tk = null;
if (!string.IsNullOrEmpty(grammar.StartBlockComment))
{
block_tk = TryLexKeyword(grammar.StartBlockComment, grammar, mapping, source, ref offset);
}
if (block_tk != null)
{
int original = offset;
while (offset < source.Length && !StartsWith(grammar, source, offset, grammar.EndBlockComment))
{
offset++;
}
if (offset >= source.Length)
{
offset = before_block_tk;
return(null);
}
else
{
var tk = new Token(Token.WhitespaceName,
block_tk.Source + source.Substring(original, offset - original) + source.Substring(offset, grammar.EndBlockComment.Length),
block_tk.Location);
offset += grammar.EndBlockComment.Length;
return(tk);
}
}
else
{
return(null);
}
}
}
}
public WhitespaceLexer(LineColumnMapping mapping, string source) : base(mapping, source)
{
}
public CommentLexer(LineColumnMapping mapping, string source) : base(mapping, source)
{
}
public EofLexer(LineColumnMapping mapping, string source) : base(mapping, source)
{
}
public StringLiteralLexer(LineColumnMapping mapping, string source) : base(mapping, source)
{
}
public OperatorLexer(LineColumnMapping mapping, string source) : base(mapping, source)
{
}
public IdentifierLexer(LineColumnMapping mapping, string source) : base(mapping, source)
{
}
