private async Task <Unit> MainWithArgs(CliOptions options)
{
Task <Unit> resultEffect = Task.FromResult(Unit.Default);
if (File.Exists(options.FilePath))
{
using var fileReader = new SafeStreamReader(options.FilePath, Encoding.UTF8);
var tokens = Lexer.Tokenize(fileReader).ToList();
// TODO: add additional cli arguments to indicate the compilation result
// TODO: add additional cli arguments to indicate the output file
var tokensString = new StringBuilder();
// string.Join(
// "\n",
// tokens.Select(t =>
// string.Join(", ",
// new []
// {
// "Lexeme: " + (t.Lexeme == "\n" ? @"\n" : t.Lexeme),
// $"Token: {t.GetType().Name}",
// // $"Absolute position: {t.AbsolutePosition}",
// $"Line number: {t.LineNumber}",
// $"Position in line: {t.PositionInLine}",
// }
// )
// )
// );
foreach (var token in tokens)
{
switch (token)
{
case NewLineSymbolToken t:
tokensString.Append("\n");
break;
case IdentifierToken t:
case LiteralToken _:
tokensString.Append($"{{{token.GetType().Name} [{token.Lexeme}] ({token.LineNumber}, {token.PositionInLine}, {token.AbsolutePosition})}} ");
break;
default:
tokensString.Append($"{{{token.GetType().Name} ({token.LineNumber}, {token.PositionInLine}, {token.AbsolutePosition})}} ");
break;
}
}
var ast = ProgramNode.Parse(tokens);
resultEffect = ast
.MapLeft(
e => FConsole.WriteLine($"An error occurred during parsing: {e.Message}")
)
.Map(SemAnalyzer.Analyze)
.Map(errors => errors
.Fold(
FConsole.WriteLine(
errors.Count > 0 ?
"Semantic errors:" :
"No semantic errors occured!"
),
(_, e) => FConsole.WriteLine($"{e.Message}")
)
)
.Match(
Left: e => e,
Right: e => e
);
}
else
{
resultEffect =
FConsole.WriteLine($"Error: there is no file {options.FilePath}");
}
return(await resultEffect);
}
public IEnumerable <IToken> Tokenize(SafeStreamReader source)
{
uint lineNumber = 1;
uint lexemeStartPositionInLine = 1;
uint absolutePosition = 1;
var maybeCurrentChar = Option <int> .None;
var currentLexemeBuffer = new StringBuilder();
var maybeToken = Option <IToken> .None;;
while ((maybeCurrentChar = source.Read()).IsSome)
{
var currentChar = maybeCurrentChar.Value();
maybeToken = Option <IToken> .None;
switch (currentChar)
{
case var c when string.IsNullOrWhiteSpace(char.ConvertFromUtf32(c)):
// if a whitespace was encountered - strip it
// and yield whatever in the buffer to the output
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
switch (c)
{
case '\r':
yield return(source.Read()
.Some <IToken>(cn =>
cn == '\n' ?
(IToken) new NewLineSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
) :
(IToken) new UnrecognizedToken(
$"\r{cn}",
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)
)
.None(new UnrecognizedToken(
$"\r",
absolutePosition,
lineNumber,
lexemeStartPositionInLine
))
);
absolutePosition += 2;
lineNumber += 1;
lexemeStartPositionInLine = 1;
break;
case '\n':
yield return(new NewLineSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
));
absolutePosition += 1;
lineNumber += 1;
lexemeStartPositionInLine = 1;
break;
default:
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
}
break;
case '.':
var currentLexeme = currentLexemeBuffer.ToString();
var maybeBeforeToken =
IntegerLiteralToken.FromString(
currentLexeme,
absolutePosition,
lineNumber,
lexemeStartPositionInLine
) ||
IdentifierToken.FromString(
currentLexeme,
absolutePosition,
lineNumber,
lexemeStartPositionInLine
) ||
UnrecognizedToken.FromString(
currentLexeme,
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)
;
var tokes =
source.Peek()
.Some <ImmutableList <IToken> >(c =>
{
var result = ImmutableList <IToken> .Empty;
IToken tokenToAdd = null;
switch (c)
{
case var _ when IsDigit(char.ConvertFromUtf32(c)):
currentLexemeBuffer.Append('.');
return(ImmutableList <IToken> .Empty);
case '.':
absolutePosition += maybeBeforeToken
.Map(t => (uint)t.Lexeme.Length)
.IfNone(0);
lexemeStartPositionInLine += maybeBeforeToken
.Some(t => (uint)t.Lexeme.Length)
.None(0u);
tokenToAdd = new RangeSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
result = maybeBeforeToken
.ToImmutableList()
.Add(tokenToAdd);
source.Read();
currentLexemeBuffer.Clear();
lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
absolutePosition += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
return(result);
default:
absolutePosition += maybeBeforeToken
.Map(t => (uint)t.Lexeme.Length)
.IfNone(0);
lexemeStartPositionInLine += maybeBeforeToken
.Some(t => (uint)t.Lexeme.Length)
.None(0u);
tokenToAdd = new DotSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
result = maybeBeforeToken
.ToImmutableList()
.Add(tokenToAdd);
currentLexemeBuffer.Clear();
lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
absolutePosition += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
return(result);
}
})
.None(() =>
{
absolutePosition += maybeBeforeToken
.Map(t => (uint)t.Lexeme.Length)
.IfNone(0);
lexemeStartPositionInLine += maybeBeforeToken
.Some(t => (uint)t.Lexeme.Length)
.None(0u);
var tokenToAdd = new DotSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
var result = maybeBeforeToken
.ToImmutableList()
.Add(tokenToAdd);
currentLexemeBuffer.Clear();
lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
absolutePosition += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
return(result);
})
;
foreach (var token in tokes)
{
yield return(token);
}
break;
case '/':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(source.Peek()
.Some <IToken>(c =>
{
switch (c)
{
case '/':
var commentContent = source.ReadLine();
var commentToken = new CommentToken(
$"/{commentContent}",
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
absolutePosition += (uint)commentContent.Length;
lineNumber += 1;
lexemeStartPositionInLine = 0;
return commentToken;
case '=':
var notEqualsToken = new NotEqualsOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
source.Read();
absolutePosition += 1;
lexemeStartPositionInLine = 1;
return notEqualsToken;
default:
return new DivideOperatorToken(
(uint)source.BaseStream.Position,
lineNumber,
lexemeStartPositionInLine
);
}
})
.None(() => new DivideOperatorToken(
(uint)source.BaseStream.Position,
lineNumber,
lexemeStartPositionInLine
)));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
case ':':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(source.Peek()
.Filter(c => c == '=')
.Some <IToken>(c =>
{
var result = new AssignmentOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
source.Read();
absolutePosition += 1;
lexemeStartPositionInLine += 1;
return result;
})
.None(new ColonSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
case '>':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(source.Peek()
.Filter(c => c == '=')
.Some <IToken>(_ =>
{
var result = new GeOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
source.Read();
absolutePosition += 1;
lexemeStartPositionInLine += 1;
return result;
})
.None(new GtOperatorToken(
(uint)absolutePosition,
lineNumber,
lexemeStartPositionInLine
)));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
case '<':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(source.Peek()
.Filter(c => c == '=')
.Some <IToken>(_ =>
{
var result = new LeOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
source.Read();
absolutePosition += 1;
lexemeStartPositionInLine += 1;
return result;
})
.None(new LtOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
case '*':
case '%':
case '+':
case '-':
case '=':
case ',':
case '[':
case ']':
case '(':
case ')':
case ';':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(SymbolLexemes
.TryGetValue(((char)currentChar).ToString())
.Some(cons => cons(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
))
.None(() => new UnrecognizedToken(
currentChar.ToString(),
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)
));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
default:
currentLexemeBuffer.Append(char.ConvertFromUtf32(currentChar));
break;
}
}
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
}
