private static Lexem ScanNumber(ILexerReader reader)
{
var startPosition = reader.Position;
var stringBuilder = new StringBuilder();
var decimalExists = false;
while (reader.Peek().Match(false, char.IsDigit) || reader.Peek().Match(false, c => c == '.'))
{
var digit = reader.Read();
var isDot =
from d in digit
select d == '.';
if (isDot.Match(false, p => p))
{
if (decimalExists)
{
throw new Exception("Multiple dots in decimal number");
}
decimalExists = true;
}
stringBuilder.Append(digit.Match(' ', d => d));
}
var maybeDouble = stringBuilder.ToString().TryParseDouble();
var parsedDouble = maybeDouble.Match(
none: () => throw new Exception("Could not parse number: " + stringBuilder),
some: d => d
);
return(new Lexem(new NumberToken(parsedDouble), new Position(startPosition, reader.Position - startPosition)));
}
private Option <Lexem> SelectLexerRule(ILexerReader reader, List <Lexem> context)
{
return(_lexerRules
.GetRules()
.Where(rule => rule.IsActive(context))
.OrderByDescending(GetRuleWeight)
.Select(rule => rule.Match(reader))
.FirstOrDefault(HasRuleMatched));
}
public Option <Lexem> Match(ILexerReader reader)
{
var predicate =
from nextCharacter in reader.Peek()
select SymbolPredicate(nextCharacter);
return(predicate.Match(false, p => p)
? Option.Some(CreateToken(reader))
: Option <Lexem> .None());
}
private static Lexem ScanIdentifier(ILexerReader reader)
{
var startPosition = reader.Position;
var stringBuilder = new StringBuilder();
while (reader.Peek().Match(false, char.IsLetterOrDigit))
{
stringBuilder.Append(reader.Read().Match(' ', c => c));
}
return(new Lexem(new IdentifierToken(stringBuilder.ToString()), new Position(startPosition, reader.Position - startPosition)));
}
private static Lexem ConsumeWhiteSpace(ILexerReader reader)
{
var startPosition = reader.Position;
while (reader.Peek().Match(false, char.IsWhiteSpace))
{
// we are not interested in what kind of whitespace, so we just discard the result
reader.Read();
}
return(new Lexem(new WhiteSpaceToken(), new Position(startPosition, reader.Position - startPosition)));
}
private Lexem ConsumeWord(ILexerReader reader)
{
var startPosition = reader.Position;
var word = new StringBuilder();
while (reader.Peek().Match(false, char.IsLetter))
{
reader.Read().AndThen(c => word.Append(c));
}
return(new Lexem(new WordToken(word.ToString()), new Position(startPosition, reader.Position - startPosition)));
}
private Lexem ConsumeRest(ILexerReader reader)
{
var startPosition = reader.Position;
var unmatchedText = new StringBuilder();
while (reader.Peek().Match(false, NotWhiteSpace))
{
reader.Read().AndThen(c => unmatchedText.Append(c));
}
return(new Lexem(new UnmatchedToken(unmatchedText.ToString()), new Position(startPosition, reader.Position - startPosition)));
}
private Lexem ConsumeTag(ILexerReader reader)
{
var startPosition = reader.Position;
StringBuilder stringBuilder = new StringBuilder();
while (reader.Peek().Match(false, IsLegalCharacter))
{
reader.Read().Match(none: stringBuilder, some: c => stringBuilder.Append(c));
}
return(new Lexem(new TagToken(stringBuilder.ToString()), new Position(startPosition, reader.Position - startPosition)));
}
public Option <Lexem> Match(ILexerReader reader)
{
var startPosition = reader.Position;
if (IsSymbolMatchingReader(reader) && (IsOperator() || HasWordBoundary(reader)))
{
foreach (var _ in _textSymbol)
{
reader.Read();
}
return(Option.Some(CreateToken(startPosition)));
}
return(Option <Lexem> .None());
}
private Lexem ScanCcAfterBB(ILexerReader reader)
{
var startPosition = reader.Position;
if (reader.Peek().Match(false, c => c == 'c'))
{
reader.Read();
if (reader.Peek().Match(false, c => c == 'c'))
{
reader.Read();
return(new Lexem(new CcAfterBbToken(), new Position(startPosition, reader.Position - startPosition)));
}
}
throw new NotImplementedException();
}
private Lexem HandleUnknownToken(ILexerReader reader)
{
throw new UnknownTokenException(reader.Peek(), CalculateCurrentLinePosition(reader.Position));
}
private bool IsSymbolMatchingReader(ILexerReader reader)
{
return(_textSymbol.Select((character, index) => new { character, index })
.All(t => reader.Peek(t.index).Match(false, c => c == t.character)));
}
private bool HasWordBoundary(ILexerReader reader)
{
// we do not want to extract key words in the middle of a word, so a symbol must have ended.
// Which means after a textsymbol must come something other than a digit or a letter.
return(reader.Peek(_textSymbol.Length).Match(true, NonLetterOrDigit));
}
private static Lexem CreateLambdaToken(ILexerReader reader)
{
var _ = reader.Read();
return(new Lexem(new LambdaToken(), new Position(reader.Position, 1)));
}
