public FSMMatch(bool success, N result, string value, LexerPosition position, int nodeId, LexerPosition newPosition, bool isLineEnding)
: this(success, result, new ReadOnlyMemory <char>(value.ToCharArray()), position, nodeId, newPosition, isLineEnding)
{
}
public static Node GetNode(this ExpressionType et, List <Token <TokenType> > expr, LexerPosition exprPos, ref InterpreterState state)
{
if (et.IsBlock())
{
bool hasArg = et != ExpressionType.ELSE;
List <List <Token <TokenType> > > args = new List <List <Token <TokenType> > >();
if (hasArg)
{
args = Interpreter.GetArgs(expr);
}
switch (et)
{
case ExpressionType.ELSE:
return(new ElseNode(exprPos));
case ExpressionType.IF:
args.ThrowIfNotSize(exprPos, 1);
return(new IfNode(args.First(), IfNode.IsElseIf(expr), exprPos));
case ExpressionType.LOOP:
args.ThrowIfNotSize(exprPos, 1);
return(new LoopNode(args.First(), exprPos));
case ExpressionType.WHILE:
args.ThrowIfNotSize(exprPos, 1);
return(new WhileNode(args.First(), exprPos));
case ExpressionType.FUNCTION_DEFINITION:
return(new FunctionDefinitionNode(
expr[1].StringWithoutQuotes,
args.Select(a => a.AsString()).ToArray(),
exprPos
));
}
}
else
{
switch (et)
{
case ExpressionType.IMPORT:
return(new ImportNode(expr[1].StringWithoutQuotes, exprPos));
case ExpressionType.RETURN:
return(new ReturnNode(expr.Skip(1).Reverse().Skip(1).Reverse().ToList(), exprPos));
case ExpressionType.BREAK:
return(new BreakNode(exprPos));
case ExpressionType.CONTINUE:
return(new ContinueNode(exprPos));
case ExpressionType.FUNCTION_CALL:
string name = expr[0].StringWithoutQuotes;
bool isSysFunc = state.Functions.Any(f => f.Name == name);
if (isSysFunc)
{
return(new FunctionCallNode(
state.Functions.First(f => f.Name.Equals(name)),
Interpreter.GetArgs(expr),
exprPos
));
}
else if (state.UserFuncs.Any(f => f.Name == name))
{
return(new FunctionCallNode(
state.UserFuncs.First(f => f.Name == name),
Interpreter.GetArgs(expr),
exprPos
));
}
throw new InterpreterException($"Unkown function \"{name}\" at {exprPos}");
case ExpressionType.ASSIGNMENT:
bool isArray = AssignmentNode.IsArrayAssignment(expr, out bool isIndexer);
return(new AssignmentNode(
expr[0].StringWithoutQuotes,
isIndexer ?
AssignmentNode.GetValueOfIndexerAssignment(expr) :
expr.Skip(2).Reverse().Skip(1).Reverse().ToList(),
isArray,
isIndexer,
isIndexer ? AssignmentNode.GetIndex(expr) : new List <Token <TokenType> >(),
exprPos
));
case ExpressionType.INCREMENT:
return(new IncrementNode(expr[0].StringWithoutQuotes, exprPos));
case ExpressionType.DECREMENT:
return(new DecrementNode(expr[0].StringWithoutQuotes, exprPos));
}
}
return(null);
}
public AssignmentNode(string name, List <Token <TokenType> > valueTokens, bool isArrayAssignment, bool isIndexer, List <Token <TokenType> > indexTokens, LexerPosition linePosition) : base(linePosition)
{
this.name = name;
this.valueTokens = valueTokens;
this.isArrayAssignment = isArrayAssignment;
this.isIndexer = isIndexer;
this.indexTokens = indexTokens;
}
public FSMMatch <N> Run(ReadOnlyMemory <char> source, LexerPosition lexerPosition)
{
ConsumeIgnored(source, lexerPosition);
// End of token stream
if (lexerPosition.Index >= source.Length)
{
return(new FSMMatch <N>(false));
}
// Make a note of where current token starts
var position = lexerPosition.Clone();
FSMMatch <N> result = null;
var currentNode = Nodes[0];
while (lexerPosition.Index < source.Length)
{
var currentCharacter = source.At(lexerPosition);
var currentValue = source.Slice(position.Index, lexerPosition.Index - position.Index + 1);
currentNode = Move(currentNode, currentCharacter, currentValue);
if (currentNode == null)
{
// No more viable transitions, so exit loop
break;
}
if (currentNode.IsEnd)
{
// Remember the possible match
result = new FSMMatch <N>(true, currentNode.Value, currentValue, position, currentNode.Id, lexerPosition, currentNode.IsLineEnding);
}
lexerPosition.Index++;
lexerPosition.Column++;
}
if (result != null)
{
// Backtrack
var length = result.Result.Value.Length;
lexerPosition.Index = result.Result.Position.Index + length;
lexerPosition.Column = result.Result.Position.Column + length;
if (HasCallback(result.NodeId))
{
result = Callbacks[result.NodeId](result);
}
return(result);
}
if (lexerPosition.Index >= source.Length)
{
// Failed on last character, so need to backtrack
lexerPosition.Index -= 1;
lexerPosition.Column -= 1;
}
var errorChar = source.Slice(lexerPosition.Index, 1);
var ko = new FSMMatch <N>(false, default(N), errorChar, lexerPosition, -1, lexerPosition, false);
return(ko);
}
public FSMMatch(bool success, N result, ReadOnlyMemory <char> value, LexerPosition position, int nodeId, LexerPosition newPosition)
{
Properties = new Dictionary <string, object>();
IsSuccess = success;
NodeId = nodeId;
IsEOS = false;
Result = new Token <N>(result, value, position);
NewPosition = newPosition;
}
public IncrementNode(string varName, LexerPosition linePosition) : base(linePosition)
{
variableName = varName;
}
public static T At <T>(this ReadOnlyMemory <T> memory, LexerPosition position)
{
return(memory.Span[position.Index]);
}
/// <summary>
/// Perform a simple lexical styling of the given range of content. This will *not*
/// correctly style many constructs, and it's not smart enough to understand
/// context. But it's fast enough that it can style most content while you type, and
/// it's accurate enough that it's good enough for a first pass. Various background
/// threads will be responsible for running the full parser and re-styling the content
/// more accurately afterward.
/// </summary>
/// <param name="editor">The editor that holds the content.</param>
/// <param name="startPos">The starting character position within the text to style.</param>
/// <param name="endPos">The ending character position within the text to style.</param>
/// <param name="filename">The name of the file being styled (for generating errors)</param>
public static void StyleRange(Scintilla editor, TextEditorControl owner, int startPos, int endPos, string filename)
{
try
{
// Ensure startPos/endPos actually refer to the document.
ValidateStartAndEndPos(editor, ref startPos, ref endPos);
// Get the real range of lines to style.
int startLine = editor.LineFromPosition(startPos);
int endLine = editor.LineFromPosition(endPos);
// Remove all the error/warning marks in this range.
owner.ClearLineMarks(startLine + 1, endLine - startLine + 1);
// Now find the actual document start and end of the range we're going to color.
int trueStartPos = editor.Lines[startLine].Position;
int trueEndPos = editor.Lines[endLine].EndPosition;
// Get the text itself in that range.
string textSlice = editor.GetTextRange(trueStartPos, trueEndPos - trueStartPos);
// Ask the real Smile Lexer to begin lexical analysis on it.
SmileLibInterop.Lexer lexer = new SmileLibInterop.Lexer(textSlice,
0, textSlice.Length, filename, startLine + 1, 1, true);
editor.StartStyling(trueStartPos);
Token token;
int lastPos = 0;
Token previousToken = null;
while ((token = lexer.Next()).Kind != TokenKind.EOI)
{
LexerPosition lexerPosition = token.Position;
int tokenStartPos = lexerPosition.LineStart + lexerPosition.Column - 1;
int tokenLength = lexerPosition.Length;
if (tokenStartPos > lastPos)
{
// Token somehow skipped over some content, so style it plain.
editor.SetStyling(tokenStartPos - lastPos, (int)StyleKind.Default);
editor.IndicatorClearRange(trueStartPos + lastPos, tokenStartPos - lastPos);
System.Diagnostics.Debug.WriteLine($"Warning: Lexer skipped {tokenStartPos - lastPos} characters at {lastPos}.");
}
else if (tokenStartPos < lastPos)
{
// For some reason, the previous token had too many characters in it,
// so shorten this one by a bit to make up for that mistake.
tokenLength -= lastPos - tokenStartPos;
System.Diagnostics.Debug.WriteLine($"Warning: Lexer grabbed too many characters ({lastPos - tokenStartPos} extra) at {lastPos}.");
}
if (tokenLength > 0)
{
StyleKind styleKind = GetDefaultStyleKindForToken(previousToken, token);
editor.SetStyling(tokenLength, (int)styleKind);
if (styleKind == StyleKind.Meta_Error)
{
editor.IndicatorCurrent = (int)IndicatorKind.Error;
editor.IndicatorFillRange(trueStartPos + tokenStartPos, tokenLength);
owner.SetLineMark(lexerPosition.Line, IndicatorKind.Error);
}
else
{
editor.IndicatorClearRange(trueStartPos + tokenStartPos, tokenLength);
}
}
lastPos = tokenStartPos + tokenLength;
previousToken = IsSemanticToken(token) ? token : previousToken;
}
}
catch (Exception e)
{
// Should never get here, but just in case, we swallow errors
// and hope a later pass will restyle the content better.
System.Diagnostics.Debug.WriteLine("Warning: LexicalStyler.StyleRange() crashed: " + e.Message);
}
}
public FunctionDefinitionNode(string name, string[] args, LexerPosition linePosition) : base(linePosition)
{
Name = name ?? throw new ArgumentNullException();
Args = args ?? throw new ArgumentNullException();
}
public BreakNode(LexerPosition linePosition) : base(linePosition)
{
}
/// <summary>
/// Perform a simple lexical styling of the given collection of lexical tokens.
/// </summary>
/// <param name="editor">The editor that holds the content.</param>
/// <param name="tokens">The tokens to be restyled.</param>
public static void StyleTokens(Scintilla editor, TextEditorControl owner, IEnumerable <Token> tokens)
{
try
{
bool isFirst = true;
int lastPos = 0;
int trueStartPos = 0;
Token previousToken = null;
foreach (Token token in tokens)
{
LexerPosition lexerPosition = token.Position;
int tokenStartPos = lexerPosition.LineStart + lexerPosition.Column - 1;
int tokenEndPos = tokenStartPos + lexerPosition.Length;
ValidateStartAndEndPos(editor, ref tokenStartPos, ref tokenEndPos);
int tokenLength = tokenEndPos - tokenStartPos;
if (isFirst)
{
editor.StartStyling(tokenStartPos);
trueStartPos = lastPos = tokenStartPos;
isFirst = false;
}
if (tokenStartPos > lastPos)
{
// Token somehow skipped over some content, so style it plain.
editor.SetStyling(tokenStartPos - lastPos, (int)StyleKind.Default);
editor.IndicatorClearRange(trueStartPos + lastPos, tokenStartPos - lastPos);
System.Diagnostics.Debug.WriteLine($"Warning: Lexer skipped {tokenStartPos - lastPos} characters at {lastPos}.");
}
else if (tokenStartPos < lastPos)
{
// For some reason, the previous token had too many characters in it,
// so shorten this one by a bit to make up for that mistake.
tokenLength -= lastPos - tokenStartPos;
System.Diagnostics.Debug.WriteLine($"Warning: Lexer grabbed too many characters ({lastPos - tokenStartPos} extra) at {lastPos}.");
}
if (tokenLength > 0)
{
StyleKind styleKind = GetDefaultStyleKindForToken(previousToken, token);
editor.SetStyling(tokenLength, (int)styleKind);
if (styleKind == StyleKind.Meta_Error)
{
editor.IndicatorCurrent = (int)IndicatorKind.Error;
editor.IndicatorFillRange(trueStartPos + tokenStartPos, tokenLength);
owner.SetLineMark(lexerPosition.Line, IndicatorKind.Error);
}
else
{
editor.IndicatorClearRange(trueStartPos + tokenStartPos, tokenLength);
}
}
lastPos = tokenStartPos + tokenLength;
previousToken = IsSemanticToken(token) ? token : previousToken;
}
}
catch (Exception e)
{
// Should never get here, but just in case, we swallow errors
// and hope a later pass will restyle the content better.
System.Diagnostics.Debug.WriteLine("Warning: LexicalStyler.StyleTokens() crashed: " + e.Message);
}
}
public ElseNode(LexerPosition linePosition) : base(linePosition)
{
}
public BlockNode(LexerPosition linePosition) : base(linePosition)
{
}
public WhileNode(List <Token <TokenType> > condition, LexerPosition linePosition) : base(linePosition)
{
this.condition = condition.AsString();
}
public ContinueNode(LexerPosition linePosition) : base(linePosition)
{
}
public ImportNode(string lib, LexerPosition linePosition) : base(linePosition)
{
Library = lib;
}
private T BuildTree <T>(Queue <Token <TokenType> > tokens, ref InterpreterState state, bool getNodesOnly = false) where T : BlockNode, new()
{
T root = new T(); // Create a new node
while (tokens.Count > 0)
{
TokenList expr = new TokenList();
// If we hit EOS we are done
Token <TokenType> token = tokens.Dequeue();
if (token == null || token.IsEOS)
{
#if DEBUG
WriteLineColor("EOS", Cyan);
#endif
break;
}
// While i is in bounds of tokens AND we have not hit an "expression ending" (; or {)
while (tokens.Count() > 0 && !token.IsEnding() && !token.IsEOS)
{
expr.Add(token);
token = tokens.Dequeue();
}
expr.Add(token);
// If expression starts with }, remove it
if (expr.Count > 0 && expr[0].TokenID == RBRACE)
{
expr = expr.Skip(1).ToList();
}
// Dont run empty expression
if (expr.Count == 0)
{
continue;
}
#if DEBUG
expr.TokenDump();
#endif
LexerPosition exprPos = expr[0].Position.Adjust();
state.Location = exprPos;
bool built = false;
// Loop over every expression type
foreach (ExpressionType type in Enum.GetValues(typeof(ExpressionType)).Cast <ExpressionType>())
{
if (type.Matches(expr))
{
Node node = type.GetNode(expr, exprPos, ref state);
if (type.IsBlock())
{
BlockNode tmp = (BlockNode)node;
Queue <Token <TokenType> > block = GetBlock(ref tokens);
BuildTree <RootNode>(block, ref state, getNodesOnly).Children.ForEach(c => tmp.Add(c));
node = tmp;
}
if (type.WillRun() || getNodesOnly)
{
root.Add(node);
}
if (!getNodesOnly)
{
if (node is FunctionDefinitionNode funcDefNode)
{
state.UserFuncs.Add(new UserFunction(
funcDefNode.Name,
funcDefNode.Args,
funcDefNode.Children
));
}
if (node is ImportNode impNode)
{
state.Import(impNode.Library);
}
}
built = true;
break;
}
}
if (!built) // Unknown Expression
{
throw new InterpreterException($"Unknown expression at {exprPos}");
}
}
return(root);
}
public ReturnNode(List<Token<TokenType>> valueTokens, LexerPosition linePosition) : base(linePosition)
{
this.valueTokens = valueTokens;
}
public LoopNode(List <Token <TokenType> > timesUnparsed, LexerPosition linePosition) : base(linePosition)
{
this.timesUnparsed = timesUnparsed;
}
public FunctionCallNode(UserFunction userFunc, List <List <Token <TokenType> > > args, LexerPosition linePosition) : base(linePosition)
{
func = null;
this.userFunc = userFunc;
isUserFunc = true;
this.args = args;
name = userFunc.Name;
}
public FSMMatch <N> Run(string source, LexerPosition position)
{
return(Run(new ReadOnlyMemory <char>(source.ToCharArray()), position));
}
public FunctionCallNode(_7sFunction func, List <List <Token <TokenType> > > args, LexerPosition linePosition) : base(linePosition)
{
this.func = func ?? throw new InterpreterException($"Unknown function at {linePosition}");
this.args = args;
isUserFunc = false;
name = func.Name;
}
public IfNode(List <Token <TokenType> > condition, bool isElseIf, LexerPosition linePosition) : base(linePosition)
{
this.condition = (isElseIf ? condition.Take(condition.Count - 1).ToList() : condition).AsString();
this.isElseIf = isElseIf;
}
protected Node(LexerPosition linePosition) => this.linePosition = linePosition;