private Feature ParseFeatureFile(string featureFile)
{
using (var reader = new StreamReader(featureFile))
{
var parser = new Parser();
var dialectProvider = new AugurkDialectProvider(Options.Language);
var tokenScanner = new TokenScanner(reader);
var tokenMatcher = new TokenMatcher(dialectProvider);
var document = parser.Parse(tokenScanner, tokenMatcher);
if (document.Feature == null)
{
throw new InvalidOperationException("Feature file failed to parse.");
}
var feature = document.Feature.ConvertToFeature(dialectProvider.GetDialect(document.Feature.Language, document.Feature.Location));
feature.SourceFilename = featureFile;
// change directory to feature-files directory
string savedDirectory = Directory.GetCurrentDirectory();
Directory.SetCurrentDirectory(Path.GetDirectoryName(featureFile));
feature.Description = ProcessDescription(feature.Description);
foreach (var scenario in feature.Scenarios)
{
scenario.Description = ProcessDescription(scenario.Description);
}
// reset current directory
Directory.SetCurrentDirectory(savedDirectory);
return(feature);
}
}
public static void SysmbolResolve(FormatedLine[] lsFormatedLine)
{
var kScanner = new TokenScanner(lsFormatedLine);
while (kScanner.MoveNext())
{
SymbolResolver.ResolveNode(kScanner.Current.m_kLinkedLeaf.Parent);
}
//if (token.tokenKind == LexerToken.Kind.ContextualKeyword)
//{
// tokenStyle = token.text == "value" ? textBuffer.styles.parameterStyle : textBuffer.styles.keywordStyle;
// if (token.text == "var" && token.m_kLinkedLeaf != null && (token.m_kLinkedLeaf.ResolvedSymbol == null || token.m_kLinkedLeaf.ResolvedSymbol.kind == SymbolKind.Error))
// SymbolResolver.ResolveNode(token.m_kLinkedLeaf.Parent);
// return tokenStyle;
//}
//var leaf = token.m_kLinkedLeaf;
//if (leaf != null && leaf.Parent != null)
//{
// if (token.tokenKind == LexerToken.Kind.Keyword)
// {
// if ((token.text == "base" || token.text == "this") && (leaf.ResolvedSymbol == null || leaf.m_sSyntaxError != null))
// SymbolResolver.ResolveNode(leaf.Parent);
// }
// else if (token.tokenKind == LexerToken.Kind.Identifier)
// {
// if (leaf.ResolvedSymbol == null || leaf.m_sSyntaxError != null)
// SymbolResolver.ResolveNode(leaf.Parent);
// }
//}
}
public SyntaxTreeNode_Leaf AddToken(TokenScanner pTokenScanner)
{
if (NumValidNodes < nodes.Count)
{
var reused = nodes[NumValidNodes] as SyntaxTreeNode_Leaf;
if (reused != null && reused.TryReuse(pTokenScanner))
{
++NumValidNodes;
return(reused);
}
}
var leaf = new SyntaxTreeNode_Leaf(pTokenScanner)
{
Parent = this, m_iChildIndex = NumValidNodes
};
if (NumValidNodes == nodes.Count)
{
nodes.Add(leaf);
++NumValidNodes;
}
else
{
nodes.Insert(NumValidNodes++, leaf);
for (var i = NumValidNodes; i < nodes.Count; ++i)
{
++nodes[i].m_iChildIndex;
}
}
return(leaf);
}
/// <summary>
/// Scan for the next token in the input stream.
/// </summary>
/// <returns>The next token in the input stream</returns>
Token Scan()
{
Token result = null;
do
{
Fetch();
} while (EatWhite());
if (buffer == null)
{
result = new TokenScanner(State, TokenEOT, Position);
}
else
{
Scan(ref result);
}
if (result == null)
{
if (ScanNewlines && (buffer[column] == NEWLINE))
{
result = new TokenScanner(State, TokenEOT, Position);
column++;
}
else
{
result = new TokenScanner(State, TokenError, Position);
State = StateError;
}
}
return(result);
}
public LR_SyntaxTree Build()
{
if (!TokenScanner.MoveNext())
{
return(null);
}
var kSyntexTree = new LR_SyntaxTree();
SyntaxRule_Cur = kSyntexTree.root = new SyntaxTreeNode_Rule(Parser.ParseRoot.RootID);
ParseNode_Cur = Parser.ParseRoot.Rule_Start.Parse(this);
SyntaxRule_Err = SyntaxRule_Cur;
ParseNode_Err = ParseNode_Cur;
while (ParseNode_Cur != null)
{
if (!ParseStep())
{
break;
}
}
return(kSyntexTree);
}
public static FeatureFileInfo ParseFile(string file)
{
var parser = new Parser <object>();
Feature feature;
using (var reader = new StreamReader(file)) {
var ts = new TokenScanner(reader);
try {
feature = (Feature)parser.Parse(ts);
} catch (CompositeParserException ex) {
_errorFiles.Add(new ErrorInfo {
FilePath = file,
CompositeException = ex
});
return(null);
} catch (NullReferenceException ex) {
_errorFiles.Add(new ErrorInfo {
FilePath = file,
Exception = ex
});
return(null);
}
ts = null;
}
var steps = (from scenario in feature.ScenarioDefinitions
from step in scenario.Steps
let exampleSteps = GetSteps(step, scenario)
from s in exampleSteps
select s).ToList();
var featureInfo = new FeatureFileInfo(file, steps);
return(featureInfo);
}
/// <summary>
/// Look at the next token in the input stream.
/// </summary>
/// <returns>The next token in the input stream</returns>
public Token PeekToken()
{
if (lookahead == null)
{
lookahead = (TokenScanner)Scan();
}
return(lookahead);
}
public static void SysmbolResolve(FormatedLine[] lsFormatedLine)
{
var kScanner = new TokenScanner(lsFormatedLine);
while (kScanner.MoveNext())
{
SymbolResolver.ResolveNode(kScanner.Current.m_kLinkedLeaf.Parent);
}
}
bool CheckSymbol(TokenId sysmbol, Regex re, ref Token output)
{
Match match;
var result = (match = re.Match(buffer, column)).Success;
if (result)
{
var value = match.Groups["text"].Value;
output = new TokenScanner(State, sysmbol, Position, value);
column += value.Length;
}
return(result);
}
public void TokenScanner_CppStyleComment()
{
string inputQuery = @"EVALUATE T -- Comment";
var tokenScanner = new TokenScanner(inputQuery, 0);
var enumerator = tokenScanner.GetEnumerator();
VerifyNextToken(enumerator, Tokens.KWEVALUATE);
VerifyNextToken(enumerator, Tokens.LEX_WHITE);
VerifyNextToken(enumerator, Tokens.TABLENAME);
VerifyNextToken(enumerator, Tokens.LEX_WHITE);
VerifyNextToken(enumerator, Tokens.LEX_COMMENT);
//? Why there is nothing after the comment?
Assert.IsFalse(enumerator.MoveNext());
}
/// <summary>
/// Consume the next token in de input stream.
/// </summary>
/// <returns>The next token in the input stream.</returns>
public Token GetToken()
{
Token result;
if (lookahead == null)
{
result = Scan();
}
else
{
result = lookahead;
lookahead = null;
}
return(result);
}
public SyntaxTreeBuilder_CSharp(TokenScanner tokenScanner, Scope_CompilationUnit pComplilationUnitScop)
{
m_kTokenScanner = tokenScanner;
ComplilationUnitScop = pComplilationUnitScop;
tokenScanner.Reset();
tokenScanner.OnTokenMoveAt = OnTokanMoveAt;
tokenScanner.EOF = new LexerToken(LexerToken.Kind.EOF, string.Empty)
{
tokenId = m_kParser.tokenEOF
};
//lines = formatedLines;
//if (EOF == null)
// EOF = new LexerToken(LexerToken.Kind.EOF, string.Empty) { tokenId = parser.tokenEOF };
}
public bool TryReuse(TokenScanner pTokenScanner)
{
if (token == null)
{
return(false);
}
var current = pTokenScanner.Current;
if (current.m_kLinkedLeaf == this)
{
token.m_kLinkedLeaf = this;
return(true);
}
return(false);
}
public void TokenScanner_CStyleCommentOneLine()
{
string inputQuery = @"EVALUATE T /* */";
var tokenScanner = new TokenScanner(inputQuery, 0);
var enumerator = tokenScanner.GetEnumerator();
VerifyNextToken(enumerator, Tokens.KWEVALUATE);
VerifyNextToken(enumerator, Tokens.LEX_WHITE);
VerifyNextToken(enumerator, Tokens.TABLENAME);
VerifyNextToken(enumerator, Tokens.LEX_WHITE);
VerifyNextToken(enumerator, Tokens.LEX_COMMENT);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsFalse(enumerator.MoveNext());
}
public void Compile(IPositionableInputStream input, TextWriter output, ILog log)
{
this.log = log;
this.offsetHistory = new Stack <int> ();
this.protectedRegions = new List <Tuple <int, int> > ();
this.scopeStructures = new Dictionary <IExpression <int>, ScopeStructure <int> > ();
TokenScanner tokenScanner = new TokenScanner(input);
if (!tokenScanner.MoveNext())
{
return;
}
Match <Token> match;
IExpression <int> expression = parser.Parse(tokenScanner, out match);
if (!match.Success)
{
log.AddError(match.Error);
return;
}
if (!tokenScanner.IsAtEnd && tokenScanner.Current.Type != TokenType.EndOfStream)
{
AddNotReachedEnd(tokenScanner.Current);
return;
}
if (log.ErrorCount == 0)
{
this.currentOffset = 0;
// DeclareExternASMCLabels(ExecuteLayoutPass<TextWriter> (expression, null,output), output);
ExecuteLayoutPass <TextWriter>(expression, null, output);
}
if (log.ErrorCount == 0)
{
this.currentOffset = 0;
ExecuteWritePass(output, expression, null);
}
}
private static void BerpGherkinParserTest(Parser parser, string file)
{
Feature feature;
using (StreamReader reader = new StreamReader(file))
{
var scanner = new TokenScanner(new FastSourceReader(reader));
//var scanner = new TokenScanner(new DefaultSourceReader(reader));
feature = (Feature)parser.Parse(scanner);
}
#if DEBUG
//Console.WriteLine(File.ReadAllText(file));
//Console.WriteLine("--------------");
var formatter = new FeatureFormatter();
Console.WriteLine(formatter.GetFeatureText(feature));
#endif
}
public void TokenScanner_OneLine()
{
string inputQuery = @"EVALUATE ROW(""a"", 1)";
var tokenScanner = new TokenScanner(inputQuery, 0);
var enumerator = tokenScanner.GetEnumerator();
VerifyNextToken(enumerator, Tokens.KWEVALUATE);
VerifyNextToken(enumerator, Tokens.LEX_WHITE);
VerifyNextToken(enumerator, Tokens.FUNCTION);
VerifyNextToken(enumerator, (Tokens)'(');
VerifyNextToken(enumerator, Tokens.STRING);
VerifyNextToken(enumerator, (Tokens)',');
VerifyNextToken(enumerator, Tokens.LEX_WHITE);
VerifyNextToken(enumerator, Tokens.NUMBER);
VerifyNextToken(enumerator, (Tokens)')');
Assert.IsTrue(enumerator.MoveNext());
Assert.IsFalse(enumerator.MoveNext());
}
protected override void Scan(ref Token result)
{
switch (State)
{
case States.NORMAL:
switch (buffer[column])
{
case '[': SetSymbol(TokenId.BRACKET_OPEN, ref result); break;
case ']': SetSymbol(TokenId.BRACKET_CLOSE, ref result); break;
case '(': SetSymbol(TokenId.PARENTESES_OPEN, ref result); break;
case ')': SetSymbol(TokenId.PARENTESES_CLOSE, ref result); break;
case '^': SetSymbol(TokenId.PWR, ref result); break;
case '/': SetSymbol(TokenId.DIV, ref result); break;
case '#': SetSymbol(TokenId.NAME, ref result, "#"); break;
default:
if (CheckSymbol(TokenId.INT, INT, ref result))
{
break;
}
CheckSymbol(TokenId.NAME, NAME, ref result);
break;
}
break;
case States.ERROR:
result = new TokenScanner(State, TokenId._ERROR_, Position, "Scanner no token recognised.");
break;
default:
result = new TokenScanner(State, TokenId._ERROR_, Position, $"Scanner state not recognised: {State}");
SetState((int)States.ERROR);
break;
}
}
public void Assemble(IPositionableInputStream input, BinaryWriter output, ILog log)
{
var assemblyContext = new AssemblyContext <T>(log, output);
var scanner = new TokenScanner(input);
if (!scanner.MoveNext())
{
return;
}
Match <Token> match;
var tree = parser.Parse(scanner, out match);
if (!match.Success)
{
log.AddError(match.Error);// + " " + inputStream.PeekOriginalLine()
return;
}
if (scanner.IsAtEnd)
{
log.AddError("Consumed all input. Shouldn't have happened.");
return;
}
if (scanner.Current.Type != TokenType.EndOfStream)
{
log.AddError(scanner.Current.Position + ": Didn't reach end, currently at " + scanner.Current);
return;
}
assemblyContext.CurrentOffset = (int)output.BaseStream.Position;
foreach (var item in FirstPass(tree, assemblyContext))
{
assemblyContext.AddCodeData(item.code, item.offset, item.template);
}
assemblyContext.CurrentOffset = (int)output.BaseStream.Position;
SecondPass(tree, assemblyContext);
}
public SyntaxTreeNode_Leaf(TokenScanner pTokenScanner)
{
token = pTokenScanner.Current;
token.m_kLinkedLeaf = this;
}
void SetSymbol(TokenId symbol, ref Token output, string value = null)
{
output = new TokenScanner(State, symbol, Position, value);
column += symbol.Text(as_symbol: true).Length;
}
public bool ParseStep()
{
if (ParseNode_Cur == null)
{
return(false);
}
var token = TokenScanner.Current;
if (ErrorMessage == null)
{
while (ParseNode_Cur != null)
{
ParseNode_Cur = ParseNode_Cur.Parse(this);
if (ErrorMessage != null || token != TokenScanner.Current)
{
break;
}
}
if (ErrorMessage == null && token != TokenScanner.Current)
{
SyntaxRule_Err = SyntaxRule_Cur;
ParseNode_Err = ParseNode_Cur;
}
}
if (ErrorMessage != null)
{
if (token.tokenKind == LexerToken.Kind.EOF)
{
return(false);
}
var missingParseTreeNode = SyntaxRule_Cur;
var missingParseNode = ParseNode_Cur;
SyntaxRule_Cur = SyntaxRule_Err;
ParseNode_Cur = ParseNode_Err;
if (SyntaxRule_Cur != null)
{
var cpt = SyntaxRule_Cur;
for (var i = cpt.NumValidNodes; i > 0 && !cpt.ChildAt(--i).HasLeafs();)
{
cpt.InvalidateFrom(i);
}
}
if (ParseNode_Cur != null)
{
int numSkipped;
ParseNode_Cur = ParseNode_Cur.Recover(this, out numSkipped);
}
if (ParseNode_Cur == null)
{
if (token.m_kLinkedLeaf != null)
{
token.m_kLinkedLeaf.ReparseToken();
}
new SyntaxTreeNode_Leaf(TokenScanner);
if (cachedErrorParseNode == ParseNode_Err)
{
token.m_kLinkedLeaf.m_sSyntaxError = cachedErrorMessage;
}
else
{
token.m_kLinkedLeaf.m_sSyntaxError = "Unexpected token! Expected " + ParseNode_Err.FirstSet.ToString(Parser.ParseRoot);
cachedErrorMessage = token.m_kLinkedLeaf.m_sSyntaxError;
cachedErrorParseNode = ParseNode_Err;
}
ParseNode_Cur = ParseNode_Err;
SyntaxRule_Cur = SyntaxRule_Err;
if (!TokenScanner.MoveNext())
{
return(false);
}
ErrorMessage = null;
}
else
{
if (missingParseNode != null && missingParseTreeNode != null)
{
SyntaxRule_Cur = missingParseTreeNode;
ParseNode_Cur = missingParseNode;
}
InsertMissingToken(ErrorMessage ?? ("Expected " + missingParseNode.FirstSet.ToString(Parser.ParseRoot)));
if (missingParseNode != null && missingParseTreeNode != null)
{
ErrorMessage = null;
ErrorToken = null;
SyntaxRule_Cur = missingParseTreeNode;
ParseNode_Cur = missingParseNode;
ParseNode_Cur = missingParseNode.parent.NextAfterChild(missingParseNode, this);
}
ErrorMessage = null;
ErrorToken = null;
}
}
return(true);
}
// multiline scan with nesting.
protected Token ScanMultiline(object id, string delimiter_open, string delimiter_close, bool ignoreCase = false)
{
Token result = null;
if (string.IsNullOrEmpty(delimiter_close))
{
result = new TokenScanner(State, id, Position);
}
else
{
if (string.IsNullOrEmpty(delimiter_open))
{
delimiter_open = null;
}
var nests = 0;
var p = Position;
var value = new StringBuilder();
var start = column;
do
{
if (buffer.Length <= column)
{
if (start < column)
{
value.Append(buffer.Substring(start));
}
if (!IncludeNewlines)
{
value.Append(NEWLINE);
}
column = buffer.Length;
Fetch();
if (buffer == null)
{
result = new TokenScanner(State, TokenError, p, $"Multiline token: In '{id}'...; found EOT before a closing delimiter '{delimiter_close}'");
}
}
else if (
(delimiter_open != null) &&
(
!ignoreCase && (buffer[column] == delimiter_open[0]) ||
ignoreCase && (char.ToUpperInvariant(buffer[column]) == char.ToUpperInvariant(delimiter_open[0]))
) &&
(delimiter_open.Length <= buffer.Length - column) &&
(string.Compare(buffer.Substring(column, delimiter_open.Length), delimiter_open, ignoreCase) == 0)
)
{
nests++;
column += delimiter_open.Length;
}
else if (
(
!ignoreCase && (buffer[column] == delimiter_close[0]) ||
ignoreCase && (char.ToUpperInvariant(buffer[column]) == char.ToUpperInvariant(delimiter_close[0]))
) &&
(delimiter_close.Length <= buffer.Length - column) &&
(string.Compare(buffer.Substring(column, delimiter_close.Length), delimiter_close, ignoreCase) == 0)
)
{
if (0 < nests)
{
nests--;
column += delimiter_open.Length;
}
else
{
var l = column - start;
if (0 < l)
{
value.Append(buffer.Substring(start, l));
}
column += delimiter_close.Length;
result = new TokenScanner(State, id, p, value.ToString());
}
}
else
{
column++;
}
}while (result == null);
}
return(result);
}
