public void ListIncludesNull()
{
var list = new List <string>(new[] { "1", "2", null });
var sut = new PeekableEnumerator <string>(list.GetEnumerator());
CompareListToPeekableEnumerator(list, sut);
}
public void EnumerateYieldReturn()
{
var list = new List <string>(new[] { "1", "2", "3" });
var sut = new PeekableEnumerator <string>(new YieldReturnEnum().GetEnumerator());
CompareListToPeekableEnumerator(list, sut);
}
public void PeekNext()
{
var list = new List <string>(new[] { "1", "2", "3" });
var sut = new PeekableEnumerator <string>(list.GetEnumerator());
CompareListToPeekableEnumerator(list, sut);
}
//public int dims;
/// <summary>
/// 此方法用来进行语法分析，调用之前须确保Parser对象里的Tokens链表不为空且不含错误token
/// </summary>
public void SyntaxAnalyze()
{ //语法分析主程序
if (Tokens.Count == 0)
{
return;
}
CurrentToken = Tokens.First();
try
{
SyntaxTree = new TreeNode(StmtType.Program);
_errorInfo = new StringBuilder();
_enumerator = Tokens.GetEnumerator().ToPeekable();
BlockLevel = 0;
SyntaxTree.LeftNode = ParseStmtSeq();
if (_errorInfo.Length != 0)
{
throw new ParseException(_errorInfo.ToString());
}
}
catch (ParseException e)
{
IsParseError = true;
Error = e.Message;
}
}
public void WrapEmptyList()
{
var list = new List <string>();
var sut = new PeekableEnumerator <string>(list.GetEnumerator());
Assert.IsNull(sut.Current);
Assert.IsNull(((IEnumerator)sut).Current);
Assert.IsNull(sut.PeekNext);
Assert.IsFalse(sut.MoveNext());
sut.Reset();
Assert.IsNull(sut.Current);
}
public void WrapList()
{
var list = new List <string>(new[] { "1", "2", "3" });
var sut = new PeekableEnumerator <string>(list.GetEnumerator());
Assert.AreEqual(3, list.Count, "This function assumes a list with 3 elements");
Assert.IsNull(sut.Current);
Assert.IsTrue(sut.MoveNext());
Assert.AreEqual("1", sut.Current);
Assert.AreEqual("1", ((IEnumerator)sut).Current);
Assert.IsTrue(sut.MoveNext());
Assert.AreEqual("2", sut.Current);
Assert.IsTrue(sut.MoveNext());
Assert.AreEqual("3", sut.Current);
Assert.IsFalse(sut.MoveNext());
sut.Reset();
Assert.IsNull(sut.Current);
Assert.IsTrue(sut.MoveNext());
Assert.AreEqual("1", sut.Current);
}
internal static IEnumerable <string> JoinToCompleteStatements(IEnumerable <string> lines, PythonLanguageVersion version, bool fixNewLine = true)
{
StringBuilder temp = new StringBuilder();
string prevText = null;
ParseResult? prevParseResult = null;
using (var e = new PeekableEnumerator <string>(lines)) {
bool skipNextMoveNext = false;
while (skipNextMoveNext || e.MoveNext())
{
skipNextMoveNext = false;
var line = e.Current;
if (e.HasNext)
{
temp.AppendLine(line);
}
else
{
temp.Append(line);
}
string newCode = temp.ToString();
var parser = Parser.CreateParser(new StringReader(newCode), version);
ParseResult result;
parser.ParseInteractiveCode(out result);
// if this parse is invalid then we need more text to be valid.
// But if this text is invalid and the previous parse was incomplete
// then appending more text won't fix things - the code in invalid, the user
// needs to fix it, so let's not break it up which would prevent that from happening.
if (result == ParseResult.Empty)
{
if (!String.IsNullOrWhiteSpace(newCode))
{
// comment line, include w/ following code.
prevText = newCode;
prevParseResult = result;
}
else
{
temp.Clear();
}
}
else if (result == ParseResult.Complete)
{
yield return(FixEndingNewLine(newCode, fixNewLine));
temp.Clear();
prevParseResult = null;
prevText = null;
}
else if (ShouldAppendCode(prevParseResult, result))
{
prevText = newCode;
prevParseResult = result;
}
else if (prevText != null)
{
// we have a complete input
yield return(FixEndingNewLine(prevText, fixNewLine));
temp.Clear();
// reparse this line so our state remains consistent as if we just started out.
skipNextMoveNext = true;
prevParseResult = null;
prevText = null;
}
else
{
prevParseResult = result;
}
}
}
if (temp.Length > 0)
{
yield return(FixEndingNewLine(temp.ToString(), fixNewLine));
}
}
internal static IEnumerable<string> JoinCodeLines(IEnumerable<string> lines, PythonLanguageVersion version) {
StringBuilder temp = new StringBuilder();
string prevText = null;
ParseResult? prevParseResult = null;
using (var e = new PeekableEnumerator<string>(lines)) {
bool skipNextMoveNext = false;
while (skipNextMoveNext || e.MoveNext()) {
skipNextMoveNext = false;
var line = e.Current;
if (e.HasNext) {
temp.AppendLine(line);
} else {
temp.Append(line);
}
string newCode = temp.ToString();
var parser = Parser.CreateParser(new StringReader(newCode), version);
ParseResult result;
parser.ParseInteractiveCode(out result);
// if this parse is invalid then we need more text to be valid.
// But if this text is invalid and the previous parse was incomplete
// then appending more text won't fix things - the code in invalid, the user
// needs to fix it, so let's not break it up which would prevent that from happening.
if (result == ParseResult.Empty) {
if (!String.IsNullOrWhiteSpace(newCode)) {
// comment line, include w/ following code.
prevText = newCode;
prevParseResult = result;
} else {
temp.Clear();
}
} else if (result == ParseResult.Complete) {
yield return FixEndingNewLine(newCode);
temp.Clear();
prevParseResult = null;
prevText = null;
} else if (ShouldAppendCode(prevParseResult, result)) {
prevText = newCode;
prevParseResult = result;
} else if (prevText != null) {
// we have a complete input
yield return FixEndingNewLine(prevText);
temp.Clear();
// reparse this line so our state remains consistent as if we just started out.
skipNextMoveNext = true;
prevParseResult = null;
prevText = null;
} else {
prevParseResult = result;
}
}
}
if (temp.Length > 0) {
yield return FixEndingNewLine(temp.ToString());
}
}
public void ThrowsResetUnavailable()
{
var sut = new PeekableEnumerator <string>(new YieldReturnEnum().GetEnumerator());
Assert.Throws <System.NotSupportedException>(() => sut.Reset());
}
/// <summary>
/// Preliminary tokenization of the expression.
/// Tokenizes numeric values, alpha values, parentheses, commas and other tokens.
/// Any whitespace is removed.
/// </summary>
public static List<Token> Parse(string expression)
{
const char LeftParenthesis = '(';
const char RightParenthesis = ')';
const char Comma = ',';
const char NumericNegative = '-';
const char DateTimeDelimiter = '#';
var whitespaceCharacters = new[] { ' ', '\t' };
var numericCharacters = new[] { '.', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };
var identifierCharacters = new[] { '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' };
var identifierSecondaryCharacters = new[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' }; // other characters that can be used as identifiers - but cannot be a starting character
var textDelimiters = new[] { '\"', '\'' };
bool isNumericNegative = false;
bool parsingText = false;
bool parsingDateTime = false;
var tokens = new List<Token>();
var currentTokenType = TokenType.Other;
var currentToken = String.Empty;
char currentTextDelimiter = '\0';
var characterTokenType = TokenType.Other;
var expressionEnumerator = new PeekableEnumerator<char>(expression);
var characterString = String.Empty;
while (expressionEnumerator.MoveNext())
{
var tokenIsSeparateCharacter = false;
var character = expressionEnumerator.Current;
// if the character is a '-' and the subsequent character is a numeric character then this is a negative number.
// otherwise it is some other character TokenType.Other -- probably a subtraction operator.
isNumericNegative = character == NumericNegative && expressionEnumerator.CanPeek && numericCharacters.Contains(expressionEnumerator.Peek);
if (textDelimiters.Contains(character) || parsingText)
{
if (textDelimiters.Contains(character) && !parsingText) // started parsing
{
characterTokenType = TokenType.Text;
characterString = String.Empty; // consume character
currentTextDelimiter = character;
parsingText = true;
}
else if (character == currentTextDelimiter && parsingText) // finished parsing
{
characterString = String.Empty; // consume character
parsingText = false;
}
else
characterString = character.ToString();
}
else if (character == DateTimeDelimiter || parsingDateTime)
{
if (!parsingDateTime) // started parsing
{
characterTokenType = TokenType.DateTime;
characterString = String.Empty; // consume character
parsingDateTime = true;
}
else if (character == DateTimeDelimiter) // finished parsing
{
characterString = String.Empty; // consume character
parsingDateTime = false;
}
else
characterString = character.ToString();
}
else if (whitespaceCharacters.Contains(character))
{
characterTokenType = TokenType.Whitespace;
characterString = String.Empty; // consume character
}
else if (identifierCharacters.Contains(character) || (currentTokenType == TokenType.Identifier && identifierSecondaryCharacters.Contains(character)))
{
characterTokenType = TokenType.Identifier;
characterString = character.ToString();
}
else if (numericCharacters.Contains(character) || isNumericNegative)
{
characterTokenType = TokenType.Number;
characterString = character.ToString();
}
else if (character == LeftParenthesis)
{
characterTokenType = TokenType.LeftParenthesis;
characterString = character.ToString();
tokenIsSeparateCharacter = true;
}
else if (character == RightParenthesis)
{
characterTokenType = TokenType.RightParenthesis;
characterString = character.ToString();
tokenIsSeparateCharacter = true;
}
else if (character == Comma)
{
characterTokenType = TokenType.Comma;
characterString = character.ToString();
tokenIsSeparateCharacter = true;
}
else
{
characterTokenType = TokenType.Other;
characterString = character.ToString();
}
if (currentTokenType == characterTokenType && !tokenIsSeparateCharacter)
currentToken += characterString;
else
{
if (currentToken.Length > 0 || currentTokenType == TokenType.Text)
tokens.Add(new Token(currentToken, currentTokenType));
currentToken = characterString;
currentTokenType = characterTokenType;
}
}
if (currentToken.Length > 0 || currentTokenType == TokenType.Text)
tokens.Add(new Token(currentToken, currentTokenType));
return tokens;
}
/// <summary>
/// Translates the tokens into meaningful functions, operations and values.
/// </summary>
private List<TranslatedToken> TranslateTokens(List<Token> tokens, List<Variable> currentVariables)
{
var translatedTokens = new List<TranslatedToken>();
var tokensEnum = new PeekableEnumerator<Token>(tokens);
while (tokensEnum.MoveNext())
{
var token = tokensEnum.Current;
switch (token.Type)
{
case TokenType.Number:
translatedTokens.Add(new ConstructToken(Number.Parse(token.Value)));
break;
case TokenType.Identifier:
var operationForTokenIdentifier = allOperators
.Select(item => item.Operation)
.SingleOrDefault(item => item.Token.Equals(token.Value));
if (operationForTokenIdentifier != null)
translatedTokens.Add(new OperatorToken(operationForTokenIdentifier));
else
translatedTokens.Add(new ConstructToken(TranslateIdentifierToken(tokensEnum, currentVariables)));
break;
case TokenType.LeftParenthesis:
translatedTokens.Add(new LeftParenthesisToken());
break;
case TokenType.RightParenthesis:
translatedTokens.Add(new RightParenthesisToken());
break;
case TokenType.Text:
translatedTokens.Add(new ConstructToken(new Text(token.Value)));
break;
case TokenType.DateTime:
translatedTokens.Add(new ConstructToken(new DateTime(System.DateTime.Parse(token.Value))));
break;
case TokenType.Other:
var operationForToken = allOperators
.Select(item => item.Operation)
.SingleOrDefault(item => item.Token.Equals(token.Value));
if (operationForToken != null)
translatedTokens.Add(new OperatorToken(operationForToken));
else
throw new InvalidOperationException(token.Value + " in an unknown operation");
break;
default:
throw new NotImplementedException();
}
}
return translatedTokens;
}
/// <summary>
/// Translates an identifier as either a function, variable name or key word.
/// A function will match a registered function name and have a left parenthesis token following it, otherwise it is a variable.
/// </summary>
private IConstruct TranslateIdentifierToken(PeekableEnumerator<Token> tokensEnum, List<Variable> currentVariables)
{
var identifierToken = tokensEnum.Current;
var reservedWordForToken = reservedWords.SingleOrDefault(reserverWord => identifierToken == reserverWord.Word);
if (reservedWordForToken != null)
{
return reservedWordForToken.Construct;
}
else
{
if (tokensEnum.CanPeek && tokensEnum.Peek.Type == TokenType.LeftParenthesis)
{
var functionForToken = allFunctions.SingleOrDefault(aFunction => identifierToken == aFunction.Name);
if (functionForToken == null)
throw new InvalidOperationException(String.Format("Function '{0}' is undefined", identifierToken));
else
return new FunctionOperation(functionForToken, GetFunctionArguments(tokensEnum, currentVariables));
}
else
{
// ensure there is only one Variable instance for the same variable name
var variable = currentVariables.SingleOrDefault(aVariable => identifierToken == aVariable.Name);
if (variable == null)
{
var newVariable = new Variable(tokensEnum.Current.Value);
currentVariables.Add(newVariable);
return newVariable;
}
else
return variable;
}
}
}
/// <summary>
/// Enumerates through the tokens, searching for tokens XX,XX,XX where XX are arguments to the function and possibly a sub expression.
/// When a ')' is found then the end of the arguments is presumed to have been found.
/// </summary>
/// <param name="tokensEnum">Expected to be currently pointing to the name of the function. The next token SHOULD be a '('</param>
private IConstruct[] GetFunctionArguments(PeekableEnumerator<Token> tokensEnum, List<Variable> currentVariables)
{
var arguments = new List<IConstruct>();
var functionName = tokensEnum.Current.Value;
if (!(tokensEnum.MoveNext() && tokensEnum.Current.Type == TokenType.LeftParenthesis))
throw new InvalidOperationException(String.Format("{0} arguments; first token should be '(' not '{1}'", functionName, tokensEnum.Current.Value));
else if (tokensEnum.Current.Type == TokenType.LeftParenthesis && tokensEnum.CanPeek && tokensEnum.Peek.Type == TokenType.RightParenthesis)
// No arguments were specified - empty parentheses were specified
tokensEnum.MoveNext(); // consume the left parenthesis token and point it to the right parenthesis token - i.e. the end of the function
else
{
bool reachedEndOfArguments = false;
while (!reachedEndOfArguments)
{
arguments.Add(GetConstructFromTokens(GetFunctionArgumentTokens(functionName, tokensEnum, currentVariables), currentVariables));
// tokensEnum.Current will be the last token processed by GetFunctionArgumentTokens()
if (tokensEnum.Current.Type == TokenType.RightParenthesis)
reachedEndOfArguments = true;
}
}
return arguments.ToArray();
}
/// <summary>
/// Gets the function's next argument's tokens by traversing the tokens until the next , or ) is found (which is not within a function).
/// Does not return the , or ) character that terminated the argument expression - it is also consumed.
/// </summary>
/// <param name="functionName">Only used in order to provide useful exceptions / errors.</param>
/// <param name="tokensEnum">Should be pointing to the token that indicates the start of a function argument; either a ( or , character.</param>
private static List<Token> GetFunctionArgumentTokens(string functionName, PeekableEnumerator<Token> tokensEnum, List<Variable> currentVariables)
{
var argumentTokens = new List<Token> ();
int functionDepth = 0;
bool reachedEndOfArgument = false;
while (!reachedEndOfArgument && tokensEnum.MoveNext())
{
var token = tokensEnum.Current;
// found the argument's terminating comma or right parenthesis
if (functionDepth == 0 && (token.Type == TokenType.Comma || token.Type == TokenType.RightParenthesis))
reachedEndOfArgument = true;
else
{
argumentTokens.Add(token);
if (token.Type == TokenType.LeftParenthesis)
functionDepth++;
else if (token.Type == TokenType.RightParenthesis)
functionDepth--;
}
}
if (argumentTokens.Count == 0)
throw new InvalidOperationException(String.Format("{0} has an empty argument", functionName));
else if (!reachedEndOfArgument)
throw new InvalidOperationException(String.Format("{0} is missing a terminating argument character; ',' or ')'", functionName));
return argumentTokens;
}
