private void AddToken(Token.Type type)
{
idToken++;
ListToken.Add(new Token(idToken, row, column - auxiliary.Length, type, auxiliary));
auxiliary = "";
state = 0;
}
public void agregarToken(Token.Tipo tipo)
{
idToken++;
ListToken.Add(new Token(idToken, fila, tipo, auxiliarLexema));
auxiliarLexema = "";
estado = 0;
}
/// <summary>
/// Realiza a análise do arquivo
/// </summary>
/// <returns></returns>
public bool Analize(string[] file)
{
if (file == null || file.Length <= 0)
{
return(false);
}
ListToken = LexicTransducer.getTokens(file);
Position = 0;
//Linha vazia
if (ListToken == null)
{
return(true);
}
//Procedimento para tirar as quebras de linhas extras no final do arquivo
while (true)
{
if (ListToken != null && ListToken[ListToken.Count - 1].Type == TokenType.LINE_BREAK)
{
ListToken.RemoveAt(ListToken.Count - 1);
}
else
{
break;
}
}
return(true);
}
public void Scan_EmptyList_Token()
{
var token = ListToken.Scan("{}");
Assert.NotNull(token);
Assert.Equal("{}", token.Value);
Assert.Empty(token.List);
}
public void Scan_EmptyListEntry_EntryIsIgnored()
{
var token = ListToken.Scan("{a,}");
Assert.NotNull(token);
Assert.Equal("{a,}", token.Value);
Assert.Collection(token.List,
entry => Assert.Equal("a", entry));
}
public void Scan_ValidList_Token()
{
var token = ListToken.Scan("{a,bc,def}");
Assert.NotNull(token);
Assert.Equal("{a,bc,def}", token.Value);
Assert.Collection(token.List,
entry => Assert.Equal("a", entry),
entry => Assert.Equal("bc", entry),
entry => Assert.Equal("def", entry));
}
private void WriteList(ListToken list)
{
IToken precedingToken = null;
var lastParameterIndex = list.Elements.Count - 1;
for (int i = 0; i < list.Elements.Count; i++)
{
IToken element = list.Elements[i];
WriteToken(element, precedingToken);
if (i != lastParameterIndex)
{
_writer.Write(',');
}
precedingToken = element;
}
}
private static ResultCode MakeLists(IList <Token> tokens)
{
var listStack = new List <List <Token> >();
int ListDepth() => listStack.Count - 1;
for (int i = 0; i < tokens.Count; i++)
{
var token = tokens[i];
var currentType = token.Type;
if (currentType == TokenType.ListOpening)
{
listStack.Add(new List <Token>());
// Remove the ListStart token.
tokens.RemoveAt(i--);
}
else if (currentType == TokenType.ListClosing)
{
if (listStack.Count == 0)
{
return(ResultCode.ListEndWithoutStart);
}
var listToken = new ListToken(listStack[ListDepth()]);
listStack.RemoveAt(ListDepth());
if (ListDepth() > -1)
{
listStack[ListDepth()].Add(listToken);
// Remove the ListEnd token.
tokens.RemoveAt(i);
}
else
{
// This replaces the ListEnd token.
tokens[i] = listToken;
}
i--;
}
else
{
if (ListDepth() > -1)
{
listStack[ListDepth()].Add(token);
tokens.RemoveAt(i--);
}
else
{
// We don't do anything with tokens outside lists.
}
}
}
if (ListDepth() > -1)
{
return(ResultCode.MissingListEnd);
}
return(ResultCode.Ok);
}
private static ResultCode MakeOperatorGroups(ExpressionOptions options, List <Token> tokens)
{
var listStack = new Stack <List <Token> >();
listStack.Push(tokens);
var opIndices = new List <(int index, ValueToken token, OperatorDefinition definition)>();
var opShifts = new List <(int index, int shift)>();
while (listStack.Count > 0)
{
var currentTokens = listStack.Pop();
for (int j = 0; j < currentTokens.Count; j++)
{
var token = currentTokens[j];
if (token is CollectionToken collectionToken)
{
listStack.Push(collectionToken.Children);
}
}
// Gather operators so we can sort them by priority rules.
opIndices.Clear();
for (int j = 0; j < currentTokens.Count; j++)
{
var token = currentTokens[j];
if (token.Type != TokenType.Operator)
{
continue;
}
var opToken = (ValueToken)token;
var opDef = options.GetOperatorDefinition(opToken.Value);
if (opDef == null)
{
return(ResultCode.UnknownSymbol);
}
opIndices.Add((index: j, opToken, opDef));
}
opIndices.Sort((x, y) =>
{
int xPriority = x.definition?.Precedence ?? 0;
int yPriority = y.definition?.Precedence ?? 0;
// Sort types in descending order.
int priorityCompare = yPriority.CompareTo(xPriority);
if (priorityCompare != 0)
{
return(priorityCompare);
}
// Sort indices of same type in ascending order.
return(x.index.CompareTo(y.index));
});
// Merge token triplets with a center operator or
// pairs with a leading operator.
opShifts.Clear();
for (int i = 0; i < opIndices.Count; i++)
{
var(opIndex, opToken, opDef) = opIndices[i];
// Offset "opIndex" by shifts caused by previous operator merges.
for (int j = 0; j < opShifts.Count; j++)
{
var(shiftIndex, shift) = opShifts[j];
if (shiftIndex < opIndex)
{
opIndex += shift;
}
}
Token?leftToken = null;
Token?rightToken = null;
int left = opIndex - 1;
if (opDef?.Associativity != OperatorSidedness.Right)
{
if (left < 0)
{
if (opDef != null && opDef.Associativity.HasFlag(OperatorSidedness.Left))
{
return(ResultCode.OperatorMissingLeftValue);
}
}
else
{
leftToken = currentTokens[left];
}
}
if (leftToken?.Type == TokenType.Operator)
{
continue;
}
int right = opIndex + 1;
if (opDef != null && opDef.Associativity != OperatorSidedness.Left)
{
if (right >= currentTokens.Count)
{
if (opDef.Associativity.HasFlag(OperatorSidedness.Right))
{
return(ResultCode.OperatorMissingRightValue);
}
}
else
{
rightToken = currentTokens[right];
if (rightToken.Type == TokenType.Operator)
{
continue;
}
// Mitigates operators with following operators.
if (rightToken.Type == TokenType.Operator)
{
int secondRight = opIndex + 2;
if (secondRight < currentTokens.Count)
{
var subToken = new ListToken(new List <Token>(2)
{
rightToken,
currentTokens[secondRight]
});
rightToken = subToken;
currentTokens[right] = rightToken;
currentTokens.RemoveAt(secondRight);
opShifts.Add((right, -1));
opIndices.RemoveAll(x => x.index == right);
}
else
{
return(ResultCode.OperatorMissingRightValue);
}
}
}
}
int sideTokenCount = 0;
if (leftToken != null)
{
sideTokenCount++;
}
if (rightToken != null)
{
sideTokenCount++;
}
// Try to skip making a 1-item list.
int resultCount = 1 + sideTokenCount;
if (resultCount == currentTokens.Count)
{
continue;
}
var resultList = new List <Token>(resultCount);
if (leftToken != null)
{
resultList.Add(leftToken);
}
resultList.Add(opToken);
if (rightToken != null)
{
resultList.Add(rightToken);
}
int firstIndex = opIndex - (leftToken != null ? 1 : 0);
var resultToken = new ListToken(resultList);
currentTokens[firstIndex] = resultToken;
currentTokens.RemoveRange(firstIndex + 1, resultList.Count - 1);
int nextShift = 1 - resultList.Count;
opShifts.Add((opIndex, nextShift));
}
}
return(ResultCode.Ok);
}
public void Scan_MissingOpeningBrace_Null()
{
Assert.Null(ListToken.Scan("ab,cd,ef}"));
}
public void Scan_MissingClosingBrace_Exception()
{
Assert.Throws <OscLexerException>(() => ListToken.Scan("{ab,cd,ef"));
}
public ExpressionTree(ExpressionOptions options, List <Token> tokens)
{
Options = options ?? throw new ArgumentNullException(nameof(options));
Tokens = tokens ?? throw new ArgumentNullException(nameof(tokens));
MetaList = new ListToken(Tokens);
}
private void ParseListToken(ListToken listToken, Scope.Node node, Scope scope,
ImmutableDictionary <Scope, Dictionary <string, object> > symbolTable, IDictionary <string, object> dict)
{
if (listToken.Delimiters != null)
{
throw new NotImplementedException("List delimiters are not supported yet.");
}
// get the number of iterations needed
var repetitions = 0;
switch (listToken.Range)
{
// automatic
case AutomaticRange automaticRange:
throw new NotImplementedException("Automatic range notation is not supported yet.");
// fixed number
case NumericRange numericRange:
repetitions = numericRange.Repetitions;
break;
// value of some named token
case TokenRange tokenRange:
object referencedValue = FindNamedTokenValue(tokenRange.ReferencedName, scope, symbolTable);
if (referencedValue is int value)
{
repetitions = value;
}
else
{
throw new ParserException($"Could not use the non-numeric value '{referencedValue}' " +
$"of token '{tokenRange.ReferencedName}' as an repetition number.");
}
break;
}
// parse the inner scope a number of times
var expandedList = new List <object>();
for (var i = 0; i < repetitions; i++)
{
Dictionary <string, object> innerRepeatedDict = ParseScope(node.InnerScope, symbolTable);
expandedList.Add(innerRepeatedDict.First().Value);
}
// find a name
Scope.Node innerNode = node;
while (innerNode.HasInnerScope)
{
innerNode = innerNode.InnerScope.First();
}
string name = "listToken";
if (innerNode.Token is NamedToken named)
{
name = named.Name + "s";
}
dict.Add(name, expandedList);
}
