public void Test()
{
var tokens = new []
{
CilSymbolRef.Create("foo"),
CilSymbolRef.Create("bar"),
CilSymbolRef.Create("other"),
CilSymbolRef.Create(typeof(string)),
CilSymbolRef.Create(typeof(string), "foo"), // links foo and string
CilSymbolRef.Create(typeof(string), "bar"), // links bar and string
CilSymbolRef.Create(typeof(string), "other"), // links other and string
};
foreach (var token in tokens)
{
target.Link(token);
}
Assert.AreEqual(1, target.Definitions.Count());
var SYM1 = new Symbol("123");
target.Resolve(tokens[0]).Symbol = SYM1;
for (int i = 1; i != tokens.Length; ++i)
{
var context = "#" + i;
Assert.AreSame(SYM1, target.GetSymbol(tokens[i]), context);
}
}
public void Test()
{
var lParen = new Symbol("(");
var rParen = new Symbol(")");
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var grammar = new Grammar
{
Symbols =
{
lParen,
rParen,
num,
ident,
qStr
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(@"blank+"),
new Matcher(@"digit+ ('.' digit+)? | '.' digit+", num),
new Matcher(
@"(alpha | [:.!@#$%^&|?*/+*=\\_-]) (alnum | [:.!@#$%^&|?*/+*=\\_-])*",
ident),
new Matcher("'\"' ('\\\\\"' | ~'\"')* '\"'", qStr),
new Matcher(ScanPattern.CreateLiteral("("), lParen),
new Matcher(ScanPattern.CreateLiteral(")"), rParen),
}
}
}
};
var target = new DfaSimulationLexer(
" (1 (\"bar\" +))",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));
var collector = new Collector<Msg>();
target.Accept(collector);
Assert.AreEqual(
new int[] { lParen.Index, num.Index, lParen.Index, qStr.Index, ident.Index, rParen.Index, rParen.Index },
collector.Select(msg => msg.Id).ToArray());
Assert.AreEqual(
new object[] { "(", "1", "(", "\"bar\"", "+", ")", ")" },
collector.Select(msg => msg.Value).ToArray());
}
public void Test()
{
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var begin = new Symbol("begin");
var end = new Symbol("end");
var assign = new Symbol(":=");
var grammar = new Grammar
{
Symbols =
{
num,
ident,
qStr,
begin,
end,
assign
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(
@"digit+ ('.' digit+)? | '.' digit+",
num),
new Matcher(
@"[01234567]+ 'Q'",
num),
new Matcher(
@"alpha alnum*",
ident),
new Matcher(
@"
quot
~(quot | esc)*
(esc . ~(quot | esc)* )*
quot
",
qStr),
new Matcher(
ScanPattern.CreateLiteral("begin"),
begin),
new Matcher(
ScanPattern.CreateLiteral("end"),
end),
new Matcher(
ScanPattern.CreateLiteral(":="),
assign),
new Matcher(
@"blank+"),
}
}
}
};
var target = new TdfaSimulationLexer(
"b:=10Q \"foo\"",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));
var collector = new Collector<Msg>();
target.Accept(collector);
Assert.AreEqual(
new int[] { ident.Index, assign.Index, num.Index, qStr.Index },
collector.Select(msg => msg.Id).ToArray());
Assert.AreEqual(
new object[] { "b", ":=", "10Q", "\"foo\"" },
collector.Select(msg => msg.Value).ToArray());
}
public void SetUp()
{
this.originalGrammar = new Grammar();
var symbols = originalGrammar.Symbols;
this.start = symbols.Add("start");
this.prefix = symbols.Add("prefix");
this.suffix = symbols.Add("suffix");
this.term1 = symbols.Add("term1");
this.term2 = symbols.Add("term2");
this.term3 = symbols.Add("term3");
this.term4 = symbols.Add("term4");
this.term5 = symbols.Add("term5");
this.inlinedNonTerm = symbols.Add("inlinedNonTerm");
this.nestedNonTerm = symbols.Add("nestedNonTerm");
originalGrammar.Start = start;
var prod = originalGrammar.Productions.Define(start, new[] { prefix, inlinedNonTerm, suffix });
prod.Actions.Add(new ForeignAction(0, 3));
}
private static Grammar BuildGrammar(CilGrammar definition)
{
var result = new Grammar();
InitContextProvider(
result,
definition.GlobalContextProvider,
result.GlobalContextProvider);
var symbolResolver = definition.SymbolResolver;
// Define grammar tokens
foreach (var cilSymbol in symbolResolver.Definitions)
{
Symbol symbol;
if (cilSymbol.Type == typeof(Exception))
{
cilSymbol.Symbol = symbol = (Symbol)result.Symbols[PredefinedTokens.Error];
symbol.Joint.Add(
new CilSymbol
{
Type = typeof(Exception),
Symbol = symbol,
Categories = SymbolCategory.DoNotDelete | SymbolCategory.DoNotInsert
});
}
else
{
symbol = new Symbol(cilSymbol.Name)
{
Categories = cilSymbol.Categories,
Joint = { cilSymbol }
};
result.Symbols.Add(symbol);
cilSymbol.Symbol = symbol;
}
}
foreach (CilSymbolFeature<Precedence> feature in definition.Precedence)
{
var symbol = symbolResolver.GetSymbol(feature.SymbolRef);
symbol.Precedence = feature.Value;
}
foreach (CilSymbolFeature<CilContextProvider> feature in definition.LocalContextProviders)
{
var symbol = symbolResolver.GetSymbol(feature.SymbolRef);
if (symbol != null)
{
InitContextProvider(result, feature.Value, symbol.LocalContextProvider);
}
}
result.Start = symbolResolver.GetSymbol(definition.Start);
// Define grammar rules
foreach (var cilProduction in definition.Productions)
{
Symbol outcome = symbolResolver.GetSymbol(cilProduction.Outcome);
var pattern = Array.ConvertAll(cilProduction.Pattern, symbolResolver.GetSymbol);
// Try to find existing rules whith same token-signature
Production production;
if (result.Productions.FindOrAdd(outcome, pattern, out production))
{
ForeignContextRef contextRef = CreateActionContextRef(cilProduction.Context);
production.Actions.Add(new ForeignAction(pattern.Length, contextRef));
}
var action = production.Actions[0];
action.Joint.Add(cilProduction);
production.ExplicitPrecedence = cilProduction.Precedence;
}
// Create conditions to allow referencing them from matchers
foreach (CilCondition cilCondition in definition.Conditions)
{
var cond = CreateCondtion(result, cilCondition);
result.Conditions.Add(cond);
}
// Create matchers
foreach (CilCondition cilCondition in definition.Conditions)
{
var condition = ConditionFromType(result, cilCondition.ConditionType);
foreach (var cilMatcher in cilCondition.Matchers)
{
SymbolBase outcome = GetMatcherOutcomeSymbol(result, symbolResolver, cilMatcher.MainOutcome, cilMatcher.AllOutcomes);
var matcher = new Matcher(
cilMatcher.Pattern,
outcome,
#if false
context: CreateActionContextRef(cilMatcher.Context),
#endif
nextCondition: ConditionFromType(result, cilMatcher.NextConditionType),
disambiguation: cilMatcher.Disambiguation);
matcher.Joint.Add(cilMatcher);
condition.Matchers.Add(matcher);
}
}
foreach (var cilMerger in definition.Mergers)
{
var symbol = symbolResolver.GetSymbol(cilMerger.Symbol);
var merger = new Merger(symbol) { Joint = { cilMerger } };
result.Mergers.Add(merger);
}
foreach (var report in definition.Reports)
{
result.Reports.Add(report);
}
return result;
}
public AmbiguousSymbol(Symbol main, Symbol[] symbols)
{
this.MainToken = main == null ? -1 : main.Index;
this.Tokens = new ReadOnlyCollection<int>((from s in symbols select s.Index).ToArray());
}
public void Test()
{
var lParen = new Symbol("(");
var rParen = new Symbol(")");
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var grammar = new Grammar
{
Symbols = {
lParen,
rParen,
num,
ident,
qStr
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(
ScanPattern.CreateRegular(
null,
@"[ \t]+"))
{
Joint = { new CilMatcher(typeof(void)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[0-9]+(?:[.][0-9]+)? | [.][0-9]+"),
num)
{
Joint = { new CilMatcher(typeof(Num)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[a-zA-Z:.!@#$%^&|?*/+*=\\_-][a-zA-Z:\d.!@#$%^&|?*/+*=\\_-]*"),
ident)
{
Joint = { new CilMatcher(typeof(string)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[""](?: \\[""] | [^""])* [""]"),
qStr)
{
Joint = { new CilMatcher(typeof(QStr)) }
},
new Matcher(
ScanPattern.CreateLiteral("("),
lParen),
new Matcher(
ScanPattern.CreateLiteral(")"),
rParen),
}
}
}
};
var target = new BootstrapScanner(
" (1 (\"bar\" +))",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance),
null,
ExceptionLogging.Instance);
var collector = new Collector<Msg>();
target.Accept(collector);
Assert.AreEqual(
new object[] { null, new Num("1"), null, new QStr("bar"), "+", null, null },
collector.Select(msg => msg.Value).ToArray());
}