/// <summary>
/// Creates a template expression given a replacement token and parameters.
/// </summary>
/// <param name="context">Context about the expression being parsed</param>
/// <returns>a template expression</returns>
public IDocumentExpression CreateExpression(DocumentExpressionContext context)
{
if (context.Body.Count > 0)
{
throw new DocumentRenderException("template tags can't have a body", context.ReplacementKeyToken);
}
TokenReader<DocumentToken> reader = new TokenReader<DocumentToken>(context.Parameters);
DocumentToken idToken;
if (reader.TryAdvance(out idToken, skipWhitespace: true) == false)
{
throw new DocumentRenderException("missing template Id", context.ReplacementKeyToken);
}
DocumentToken evalToken;
if (reader.TryAdvance(out evalToken, skipWhitespace: true) == false)
{
throw new DocumentRenderException("missing eval token", idToken);
}
var ret = new TemplateExpression(idToken, evalToken);
return ret;
}
public void MultipleReads()
{
var queue = new TokenReader(new StringReader("abcd"));
queue.Read();
queue.Read();
var token = queue.Peek();
Assert.AreEqual('c', token.Character);
}
/// <summary>
/// Creates a clear variable expression given replacement info
/// </summary>
/// <param name="context">The context that contains information about the document being rendered</param>
/// <returns>a clear variable expression</returns>
public IDocumentExpression CreateExpression(DocumentExpressionContext context)
{
if (context.Body.Count > 0)
{
throw new DocumentRenderException("clearvar tags can't have a body", context.ReplacementKeyToken);
}
TokenReader<DocumentToken> reader = new TokenReader<DocumentToken>(context.Parameters);
DocumentToken variableName;
if (reader.TryAdvance(out variableName, skipWhitespace: true) == false)
{
throw new DocumentRenderException("Expected variable name after clearvar tag", context.ReplacementKeyToken);
}
return new ClearVarExpression(variableName);
}
public void ReadTokens()
{
var queue = new TokenReader(new StringReader("abc\\d"));
var token = queue.Read();
Assert.AreEqual('a', token.Character);
token = queue.Read();
Assert.AreEqual('b', token.Character);
token = queue.Read();
Assert.AreEqual('c', token.Character);
token = queue.Read();
Assert.AreEqual('d', token.Character);
Assert.IsTrue(token.IsEscaped);
token = queue.Read();
Assert.IsTrue(token.IsEnd);
}
public static Config Load(TextReader reader, bool ignoreDuplicates = false)
{
var sections = new List<ConfigSection>();
ConfigSection currentSection = null;
IDictionary<string, AttributeValue> values = null;
using (reader)
{
var tokenReader = new TokenReader(reader);
while (tokenReader.ReadNext())
{
if (tokenReader.IsSection())
{
if (currentSection != null)
{
sections.Add(currentSection);
}
currentSection = ConfigSection.FromName(tokenReader.GetValue());
values = new Dictionary<string, AttributeValue>();
}
else if (tokenReader.IsAttribute())
{
if (currentSection == null)
throw new InvalidOperationException("attribute value without section");
var attr = AttributeValue.LoadFromString(tokenReader.GetValue());
if (!ignoreDuplicates)
{
if (values.ContainsKey(attr.AttributeName))
throw new DuplicateNameException("attributename");
values[attr.AttributeName] = attr;
}
currentSection.AddAttribute(attr);
}
}
if (currentSection != null)
sections.Add(currentSection);
}
return new Config(sections);
}
private static Gt.ParserLibrary.Tokens.Token[] CreateTokens()
{
m_Tokens = new System.Collections.Hashtable();
m_Tokens["$TOKENS"] = new TokenReader("$TOKENS", new Gt.ParserLibrary.Tokens.State[] {
null,
new State(false, 0, new int[] {
4,
4,
4,
4,
4,
4,
4,
4,
4,
7,
4,
4,
14,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
13,
16,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
12,
15,
6,
6,
6,
6,
6,
6,
6,
6,
6,
6,
9,
10,
10,
10,
10,
10,
11,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
5,
5,
5,
5,
5,
5,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
5,
5,
5,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2}, 1, null),
new State(true, 0, new int[] {
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2}, 126, new int[] {
16}),
new State(true, 0, new int[] {
18,
0,
0,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
0,
0,
0,
0,
0,
0,
0,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
17,
17,
17,
17,
17,
17,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
3,
17,
17,
17}, 45, new int[] {
10}),
new State(true, 0, new int[] {
4,
4,
4,
4,
4,
4,
4,
4,
4,
0,
4,
4,
0,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4,
4}, 1, new int[] {
14}),
new State(true, 0, new int[] {
17,
0,
0,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
0,
0,
0,
0,
0,
0,
0,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
5,
5,
5,
5,
5,
5,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
5,
5,
5}, 45, new int[] {
12}),
new State(true, 0, new int[] {
19,
0,
0,
20,
20,
20,
20,
20,
20,
20,
20,
20,
20,
0,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19}, 45, new int[] {
11}),
new State(true, 0, null, 65535, new int[] {
5}),
new State(true, 0, new int[] {
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8}, 33, new int[] {
15}),
new State(true, 0, null, 65535, new int[] {
1}),
new State(true, 0, new int[] {
10,
10,
10,
10,
10}, 59, new int[] {
17}),
new State(true, 0, null, 65535, new int[] {
2}),
new State(true, 0, new int[] {
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8}, 33, new int[] {
6,
15}),
new State(true, 0, null, 65535, new int[] {
0}),
new State(true, 0, null, 65535, new int[] {
4}),
new State(true, 0, null, 65535, new int[] {
7}),
new State(true, 0, new int[] {
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8}, 33, new int[] {
15,
3}),
new State(true, 0, new int[] {
17,
0,
0,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
0,
0,
0,
0,
0,
0,
0,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17,
17}, 45, new int[] {
12}),
new State(true, 0, new int[] {
18,
0,
0,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
0,
0,
0,
0,
0,
0,
0,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
17,
17,
17,
17,
17,
17,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
18,
17,
17,
17}, 45, new int[] {
11}),
new State(true, 0, new int[] {
19,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
0,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19}, 45, new int[] {
11}),
new State(true, 0, new int[] {
19,
0,
0,
21,
21,
21,
21,
21,
21,
21,
21,
21,
21,
0,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19}, 45, new int[] {
11}),
new State(true, 0, new int[] {
19,
0,
0,
22,
22,
22,
22,
22,
22,
22,
22,
22,
22,
0,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19}, 45, new int[] {
9,
11}),
new State(true, 0, new int[] {
19,
0,
0,
22,
22,
22,
22,
22,
22,
22,
22,
22,
22,
0,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
0,
0,
0,
0,
0,
0,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19,
19}, 45, new int[] {
11})});
return new Gt.ParserLibrary.Tokens.Token[] {
new Gt.ParserLibrary.Tokens.Token("SPACE", "SPACE", 1, false),
new Gt.ParserLibrary.Tokens.Token("COLON", "COLON", 1, false),
new Gt.ParserLibrary.Tokens.Token("AT", "AT", 1, false),
new Gt.ParserLibrary.Tokens.Token("BANG", "BANG", 1, false),
new Gt.ParserLibrary.Tokens.Token("CR", "CR", 1, false),
new Gt.ParserLibrary.Tokens.Token("LF", "LF", 1, false),
new Gt.ParserLibrary.Tokens.Token("DOT", "DOT", 1, false),
new Gt.ParserLibrary.Tokens.Token("SLASH", "SLASH", 1, false),
new Gt.ParserLibrary.Tokens.Token("DIGITS", "DIGITS", 9, false),
new Gt.ParserLibrary.Tokens.Token("THREEDIGITS", "THREEDIGITS", 10, false),
new Gt.ParserLibrary.Tokens.Token("WORD", "WORD", 11, false),
new Gt.ParserLibrary.Tokens.Token("SHORTNAME", "SHORTNAME", 10, false),
new Gt.ParserLibrary.Tokens.Token("NICKNAME", "NICKNAME", 9, false),
new Gt.ParserLibrary.Tokens.Token("SPECIALS", "SPECIALS", 1, false),
new Gt.ParserLibrary.Tokens.Token("CONTROLS", "CONTROLS", 1, false),
new Gt.ParserLibrary.Tokens.Token("SYMBOLS", "SYMBOLS", 1, false),
new Gt.ParserLibrary.Tokens.Token("OTHERS", "OTHERS", 1, false),
new Gt.ParserLibrary.Tokens.Token("EXTRAS", "EXTRAS", 1, false)};
}
public override IEnumerator <Parser> Parse(RantCompiler compiler, CompileContext context, TokenReader reader,
Action <RST> actionCallback)
{
var blockStartToken = reader.PrevLooseToken;
var items = new List <RST>();
var actions = new List <RST>();
// "why are these not lists or arrays" i yell into the void, too lazy to find out why
List <_ <int, double> > constantWeights = null;
List <_ <int, RST> > dynamicWeights = null;
int blockNumber = 0;
void itemCallback(RST action) => actions.Add(action);
compiler.AddContext(CompileContext.BlockEndSequence);
compiler.AddContext(CompileContext.BlockSequence);
while (compiler.NextContext == CompileContext.BlockSequence)
{
// block weight
if (reader.PeekLooseToken().Type == R.LeftParen)
{
constantWeights = constantWeights ?? (constantWeights = new List <_ <int, double> >());
dynamicWeights = dynamicWeights ?? (dynamicWeights = new List <_ <int, RST> >());
var firstToken = reader.ReadLooseToken();
var sequence = new List <RST>();
void weightCallback(RST rst) => sequence.Add(rst);
compiler.SetNextActionCallback(weightCallback);
compiler.AddContext(CompileContext.BlockWeight);
yield return(Get <SequenceParser>());
// Constant
if (sequence.TrueForAll(rst => rst is RstText))
{
var sb = new StringBuilder();
foreach (var rst in sequence)
{
sb.Append((rst as RstText).Text);
}
string txt = sb.ToString();
if (!Util.ParseDouble(txt, out double doubleValue))
{
compiler.SyntaxError(reader.PrevLooseToken, false, "err-compiler-invalid-constweight");
}
else
{
constantWeights.Add(new _ <int, double>(blockNumber, doubleValue));
}
}
// Dynamic
else
{
if (sequence.Count == 0)
{
compiler.SyntaxError(firstToken, false, "err-compiler-empty-weight");
}
else
{
dynamicWeights.Add(new _ <int, RST>(blockNumber, new RstSequence(sequence, sequence[0].Location)));
}
}
}
reader.SkipSpace();
compiler.SetNextActionCallback(itemCallback);
var startToken = reader.PeekToken();
yield return(Get <SequenceParser>());
// Don't wrap single nodes in a sequence, it's unnecessary
items.Add(actions.Count == 1 ? actions[0] : new RstSequence(actions, startToken.ToLocation()));
actions.Clear();
blockNumber++;
}
compiler.LeaveContext();
compiler.SetNextActionCallback(actionCallback);
actionCallback(new RstBlock(blockStartToken.ToLocation(), items, dynamicWeights, constantWeights));
}
/// <summary>
/// 次のトークンを先読みし、指定した型であるかを判定します。
/// </summary>
/// <param name="reader">トークン リーダー</param>
/// <param name="type">先読みするトークンの種類</param>
/// <param name="trim">読み込みが合致した時にトークンを読み飛ばす</param>
/// <returns>合致すればtrue</returns>
private static bool TryLookAhead(ref TokenReader reader, Token.TokenType type, bool trim = true)
{
if (!reader.IsRemainToken)
return false;
if (reader.LookAhead().Type == type)
{
if (trim)
reader.Get();
return true;
}
else
{
return false;
}
}
public new IConceptInfo ParseNextConcept(TokenReader tokenReader, Stack<IConceptInfo> context, IEnumerable<IConceptParser> conceptParsers)
{
return base.ParseNextConcept(tokenReader, context, conceptParsers);
}
private static ClusterTuple MakeCluster(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeCluster");
var ctuple = new ClusterTuple();
ctuple.Negate = TryLookAhead(ref reader, Token.TokenType.Exclamation);
AssertNext(ref reader, Token.TokenType.OpenBracket);
if (reader.IsRemainToken && !TryLookAhead(ref reader, Token.TokenType.CloseBracket, false))
{
ctuple.InnerCluster = MakeInnerCluster(ref reader);
}
AssertNext(ref reader, Token.TokenType.CloseBracket);
AssertNextAny(ref reader, true, Token.TokenType.CloseBracket,
Token.TokenType.ConcatenatorAnd, Token.TokenType.ConcatenatorOr);
return ctuple;
}
private static FilterAttrTuple MakeFilterAttr(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeFilterAttr");
var ret = new FilterAttrTuple();
if(TryLookAhead(ref reader, Token.TokenType.Exclamation))
{
// Negate
ret.IsNegate = true;
}
// 引数を読む
ret.ArgDescript = MakeArgDescript(ref reader);
// 終わり
AssertNextAny(ref reader, false, Token.TokenType.CloseBracket,
Token.TokenType.ConcatenatorAnd, Token.TokenType.ConcatenatorOr);
return ret;
}
public void TestTokenReader17()
{
TokenReader reader = new TokenReader(CultureInfo.InvariantCulture);
List <Token> tokens = reader.Read("logn(2,5)");
#if !NETCORE
Assert.AreEqual(6, tokens.Count);
Assert.AreEqual("logn", tokens[0].Value);
Assert.AreEqual(0, tokens[0].StartPosition);
Assert.AreEqual(4, tokens[0].Length);
Assert.AreEqual('(', tokens[1].Value);
Assert.AreEqual(4, tokens[1].StartPosition);
Assert.AreEqual(1, tokens[1].Length);
Assert.AreEqual(TokenType.LeftBracket, tokens[1].TokenType);
Assert.AreEqual(2, tokens[2].Value);
Assert.AreEqual(5, tokens[2].StartPosition);
Assert.AreEqual(1, tokens[2].Length);
Assert.AreEqual(',', tokens[3].Value);
Assert.AreEqual(6, tokens[3].StartPosition);
Assert.AreEqual(1, tokens[3].Length);
Assert.AreEqual(TokenType.ArgumentSeparator, tokens[3].TokenType);
Assert.AreEqual(5, tokens[4].Value);
Assert.AreEqual(7, tokens[4].StartPosition);
Assert.AreEqual(1, tokens[4].Length);
Assert.AreEqual(')', tokens[5].Value);
Assert.AreEqual(8, tokens[5].StartPosition);
Assert.AreEqual(1, tokens[5].Length);
Assert.AreEqual(TokenType.RightBracket, tokens[5].TokenType);
#else
Assert.Equal(6, tokens.Count);
Assert.Equal("logn", tokens[0].Value);
Assert.Equal(0, tokens[0].StartPosition);
Assert.Equal(4, tokens[0].Length);
Assert.Equal('(', tokens[1].Value);
Assert.Equal(4, tokens[1].StartPosition);
Assert.Equal(1, tokens[1].Length);
Assert.Equal(TokenType.LeftBracket, tokens[1].TokenType);
Assert.Equal(2, tokens[2].Value);
Assert.Equal(5, tokens[2].StartPosition);
Assert.Equal(1, tokens[2].Length);
Assert.Equal(',', tokens[3].Value);
Assert.Equal(6, tokens[3].StartPosition);
Assert.Equal(1, tokens[3].Length);
Assert.Equal(TokenType.ArgumentSeparator, tokens[3].TokenType);
Assert.Equal(5, tokens[4].Value);
Assert.Equal(7, tokens[4].StartPosition);
Assert.Equal(1, tokens[4].Length);
Assert.Equal(')', tokens[5].Value);
Assert.Equal(8, tokens[5].StartPosition);
Assert.Equal(1, tokens[5].Length);
Assert.Equal(TokenType.RightBracket, tokens[5].TokenType);
#endif
}
public IConceptInfo ParseNextConcept(TokenReader tokenReader, Stack <IConceptInfo> context, MultiDictionary <string, IConceptParser> conceptParsers)
{
return(Invoke <IConceptInfo>("ParseNextConcept", tokenReader, context, conceptParsers));
}
public void TestTokenReader4()
{
TokenReader reader = new TokenReader();
List <Token> tokens = reader.Read("(42+ 8) *2");
#if !NETCORE
Assert.AreEqual(7, tokens.Count);
Assert.AreEqual('(', tokens[0].Value);
Assert.AreEqual(0, tokens[0].StartPosition);
Assert.AreEqual(1, tokens[0].Length);
Assert.AreEqual(42, tokens[1].Value);
Assert.AreEqual(1, tokens[1].StartPosition);
Assert.AreEqual(2, tokens[1].Length);
Assert.AreEqual('+', tokens[2].Value);
Assert.AreEqual(3, tokens[2].StartPosition);
Assert.AreEqual(1, tokens[2].Length);
Assert.AreEqual(8, tokens[3].Value);
Assert.AreEqual(5, tokens[3].StartPosition);
Assert.AreEqual(1, tokens[3].Length);
Assert.AreEqual(')', tokens[4].Value);
Assert.AreEqual(6, tokens[4].StartPosition);
Assert.AreEqual(1, tokens[4].Length);
Assert.AreEqual('*', tokens[5].Value);
Assert.AreEqual(8, tokens[5].StartPosition);
Assert.AreEqual(1, tokens[5].Length);
Assert.AreEqual(2, tokens[6].Value);
Assert.AreEqual(9, tokens[6].StartPosition);
Assert.AreEqual(1, tokens[6].Length);
#else
Assert.Equal(7, tokens.Count);
Assert.Equal('(', tokens[0].Value);
Assert.Equal(0, tokens[0].StartPosition);
Assert.Equal(1, tokens[0].Length);
Assert.Equal(42, tokens[1].Value);
Assert.Equal(1, tokens[1].StartPosition);
Assert.Equal(2, tokens[1].Length);
Assert.Equal('+', tokens[2].Value);
Assert.Equal(3, tokens[2].StartPosition);
Assert.Equal(1, tokens[2].Length);
Assert.Equal(8, tokens[3].Value);
Assert.Equal(5, tokens[3].StartPosition);
Assert.Equal(1, tokens[3].Length);
Assert.Equal(')', tokens[4].Value);
Assert.Equal(6, tokens[4].StartPosition);
Assert.Equal(1, tokens[4].Length);
Assert.Equal('*', tokens[5].Value);
Assert.Equal(8, tokens[5].StartPosition);
Assert.Equal(1, tokens[5].Length);
Assert.Equal(2, tokens[6].Value);
Assert.Equal(9, tokens[6].StartPosition);
Assert.Equal(1, tokens[6].Length);
#endif
}
public void TestTokenReader16()
{
TokenReader reader = new TokenReader(CultureInfo.InvariantCulture);
List <Token> tokens = reader.Read("5*-(2+43)");
#if !NETCORE
Assert.AreEqual(8, tokens.Count);
Assert.AreEqual(5, tokens[0].Value);
Assert.AreEqual(0, tokens[0].StartPosition);
Assert.AreEqual(1, tokens[0].Length);
Assert.AreEqual('*', tokens[1].Value);
Assert.AreEqual(1, tokens[1].StartPosition);
Assert.AreEqual(1, tokens[1].Length);
Assert.AreEqual('_', tokens[2].Value);
Assert.AreEqual(2, tokens[2].StartPosition);
Assert.AreEqual(1, tokens[2].Length);
Assert.AreEqual('(', tokens[3].Value);
Assert.AreEqual(3, tokens[3].StartPosition);
Assert.AreEqual(1, tokens[3].Length);
Assert.AreEqual(2, tokens[4].Value);
Assert.AreEqual(4, tokens[4].StartPosition);
Assert.AreEqual(1, tokens[4].Length);
Assert.AreEqual('+', tokens[5].Value);
Assert.AreEqual(5, tokens[5].StartPosition);
Assert.AreEqual(1, tokens[5].Length);
Assert.AreEqual(43, tokens[6].Value);
Assert.AreEqual(6, tokens[6].StartPosition);
Assert.AreEqual(2, tokens[6].Length);
Assert.AreEqual(')', tokens[7].Value);
Assert.AreEqual(8, tokens[7].StartPosition);
Assert.AreEqual(1, tokens[7].Length);
#else
Assert.Equal(8, tokens.Count);
Assert.Equal(5, tokens[0].Value);
Assert.Equal(0, tokens[0].StartPosition);
Assert.Equal(1, tokens[0].Length);
Assert.Equal('*', tokens[1].Value);
Assert.Equal(1, tokens[1].StartPosition);
Assert.Equal(1, tokens[1].Length);
Assert.Equal('_', tokens[2].Value);
Assert.Equal(2, tokens[2].StartPosition);
Assert.Equal(1, tokens[2].Length);
Assert.Equal('(', tokens[3].Value);
Assert.Equal(3, tokens[3].StartPosition);
Assert.Equal(1, tokens[3].Length);
Assert.Equal(2, tokens[4].Value);
Assert.Equal(4, tokens[4].StartPosition);
Assert.Equal(1, tokens[4].Length);
Assert.Equal('+', tokens[5].Value);
Assert.Equal(5, tokens[5].StartPosition);
Assert.Equal(1, tokens[5].Length);
Assert.Equal(43, tokens[6].Value);
Assert.Equal(6, tokens[6].StartPosition);
Assert.Equal(2, tokens[6].Length);
Assert.Equal(')', tokens[7].Value);
Assert.Equal(8, tokens[7].StartPosition);
Assert.Equal(1, tokens[7].Length);
#endif
}
private bool MatchesTokenBegin(string format, int position, string tokenCandidate, TokenReader canReadToken, out string realToken)
{
if (position + tokenCandidate.Length <= format.Length &&
format.ContainsAt(tokenCandidate, position) &&
canReadToken(format, position, out realToken))
{
return(true);
}
realToken = null;
return(false);
}
public TemplateHandler(TokenReader Tokens, TextWriter TextWriter)
{
this.Tokens = Tokens;
this.TextWriter = TextWriter;
this.Blocks = new Dictionary <string, ParserNode>();
}
new public IConceptInfo ParseNextConcept(TokenReader tokenReader, Stack <IConceptInfo> context, MultiDictionary <string, IConceptParser> conceptParsers)
{
return(base.ParseNextConcept(tokenReader, context, conceptParsers));
}
private JReader(IReaderAdapter d)
{
_tr = new TokenReader();
_awaitingReadValue = false;
_delegate = d;
}
public Parser(TokenReader reader)
{
_reader = reader;
}
private static ArgDescriptTuple MakeArgDescript(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeArgDescript");
ArgDescriptTuple ret = new ArgDescriptTuple();
if (TryLookAhead(ref reader, Token.TokenType.Collon))
{
// NOT EPSILON
ret.Args = MakeArgs(ref reader);
}
AssertNextAny(ref reader, false, Token.TokenType.CloseBracket,
Token.TokenType.ConcatenatorAnd, Token.TokenType.ConcatenatorOr);
return ret;
}
public IConceptInfo ParseNextConcept(TokenReader tokenReader, Stack <IConceptInfo> context, MultiDictionary <string, IConceptParser> conceptParsers)
{
return(this.Invoke(nameof(ParseNextConcept), tokenReader, context, conceptParsers).Item1);
}
private static ExpressionTuple MakeExpression(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeExpression");
ExpressionTuple ret = new ExpressionTuple();
if (TryLookAhead(ref reader, Token.TokenType.Exclamation, false) ||
TryLookAhead(ref reader, Token.TokenType.OpenBracket, false))
{
// 新クラスタの開始
ret.Cluster = MakeCluster(ref reader);
}
else
{
// フィルタの開始
ret.Filter = MakeFilter(ref reader);
}
AssertNextAny(ref reader, true, Token.TokenType.CloseBracket,
Token.TokenType.ConcatenatorAnd, Token.TokenType.ConcatenatorOr);
return ret;
}
public UnexpectedTokenException(TokenReader tokenReader) : base()
{
TokenReader = tokenReader ?? throw new ArgumentNullException(nameof(tokenReader));
}
private static RootTuple MakeRoot(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeRoot");
var ret = new RootTuple();
if (reader.IsRemainToken)
{
ret.Cluster = MakeCluster(ref reader);
}
if (reader.IsRemainToken)
throw new ArgumentException("クエリが途中で終了しています。閉じ括弧が多すぎる可能性があります。(次のクエリ:" + reader.LookAhead().ToString() +
", インデックス:" + reader.LookAhead().DebugIndex.ToString());
return ret;
}
/// <summary>
/// Deserializes the given frame given a known width and height.
/// </summary>
/// <param name="serializedFrame">the frame data</param>
/// <param name="width">the known width of the frame</param>
/// <param name="height">the known height of the frame</param>
/// <returns>a deserialized frame that's either a raw frame or a diff frame, depending on what was in the serialized string</returns>
public ConsoleBitmapFrame DeserializeFrame(string serializedFrame, int width, int height)
{
var tokens = tokenizer.Tokenize(serializedFrame);
var reader = new TokenReader <Token>(tokens);
reader.Expect("[");
var timestampToken = reader.Advance();
var timestamp = new TimeSpan(long.Parse(timestampToken.Value));
reader.Expect("]");
reader.Expect("[");
reader.Advance();
var isDiff = reader.Current.Value == "Diff";
reader.Expect("]");
if (isDiff)
{
var diffFrame = new ConsoleBitmapDiffFrame()
{
Timestamp = timestamp,
Diffs = new System.Collections.Generic.List <ConsoleBitmapPixelDiff>()
};
var lastBackground = ConsoleString.DefaultBackgroundColor;
var lastForeground = ConsoleString.DefaultForegroundColor;
while (reader.CanAdvance())
{
reader.Expect("[");
if (reader.Peek().Value.StartsWith("F=") || reader.Peek().Value.StartsWith("B="))
{
reader.Advance();
var match = Regex.Match(reader.Current.Value, @"(?<ForB>F|B)=(?<color>\w+)");
if (match.Success == false)
{
throw new FormatException($"Unexpected token {reader.Current.Value} at position {reader.Current.Position} ");
}
var isForeground = match.Groups["ForB"].Value == "F";
if (isForeground)
{
if (Enum.TryParse(match.Groups["color"].Value, out ConsoleColor c))
{
lastForeground = (RGB)c;
}
else if (RGB.TryParse(match.Groups["color"].Value, out lastForeground) == false)
{
throw new ArgumentException($"Expected a color @ {reader.Position}");
}
}
else
{
if (Enum.TryParse(match.Groups["color"].Value, out ConsoleColor c))
{
lastBackground = (RGB)c;
}
else if (RGB.TryParse(match.Groups["color"].Value, out lastBackground) == false)
{
throw new ArgumentException($"Expected a color @ {reader.Position}");
}
}
reader.Expect("]");
}
else
{
var match = Regex.Match(reader.Advance().Value, "(?<x>\\d+),(?<y>\\d+),(?<val>.+)");
if (match.Success == false)
{
throw new FormatException("Could not parse pixel diff");
}
var valGroup = match.Groups["val"].Value;
char?nextChar = valGroup.Length == 1 ? valGroup[0] : valGroup == "OB" ? '[' : valGroup == "CB" ? ']' : new char?();
if (nextChar.HasValue == false)
{
throw new FormatException($"Unexpected token {nextChar} @ {reader.Position}");
}
diffFrame.Diffs.Add(new ConsoleBitmapPixelDiff()
{
X = int.Parse(match.Groups["x"].Value),
Y = int.Parse(match.Groups["y"].Value),
Value = new ConsoleCharacter(nextChar.Value, lastForeground, lastBackground),
});
reader.Expect("]");
}
}
return(diffFrame);
}
else
{
var rawFrame = new ConsoleBitmapRawFrame()
{
Timestamp = timestamp,
Pixels = new ConsoleCharacter[width][]
};
for (var i = 0; i < width; i++)
{
rawFrame.Pixels[i] = new ConsoleCharacter[height];
}
var x = 0;
var y = 0;
var lastFg = ConsoleString.DefaultForegroundColor;
var lastBg = ConsoleString.DefaultBackgroundColor;
while (reader.CanAdvance())
{
reader.Expect("[");
var next = reader.Advance();
var match = Regex.Match(next.Value, @"(?<ForB>F|B)=(?<color>\w+)");
if (match.Success)
{
var isForeground = match.Groups["ForB"].Value == "F";
if (isForeground)
{
if (Enum.TryParse <ConsoleColor>(match.Groups["color"].Value, out ConsoleColor c))
{
lastFg = c;
}
else if (RGB.TryParse(match.Groups["color"].Value, out lastFg) == false)
{
throw new ArgumentException($"Expected a color @ {reader.Position}");
}
}
else
{
if (Enum.TryParse <ConsoleColor>(match.Groups["color"].Value, out ConsoleColor c))
{
lastBg = c;
}
else if (RGB.TryParse(match.Groups["color"].Value, out lastBg) == false)
{
throw new ArgumentException($"Expected a color @ {reader.Position}");
}
}
}
else
{
char?nextChar = next.Value.Length == 1 ? next.Value[0] : next.Value == "OB" ? '[' : next.Value == "CB" ? ']' : new char?();
if (nextChar.HasValue == false)
{
throw new FormatException($"Unexpected token {nextChar} @ {next.Position}");
}
rawFrame.Pixels[x][y++] = new ConsoleCharacter(nextChar.Value, lastFg, lastBg);
if (y == height)
{
y = 0;
x++;
}
}
reader.Expect("]");
}
return(rawFrame);
}
}
private void InternalSetCompilerAndReader(RantCompiler compiler, TokenReader reader)
{
if (compiler == null)
throw new RantInternalException($"{GetType().Name}.InternalSetCompilerAndReader(): Given compiler is null.");
if (reader == null)
throw new RantInternalException($"{GetType().Name}.InternalSetCompilerAndReader(): Given token reader is null.");
this.compiler = compiler;
this.reader = reader;
}
public abstract IEnumerator<Parser> Parse(RantCompiler compiler, CompileContext context, TokenReader reader, Action<RantAction> actionCallback);
private IEnumerable <CompletionData> GetCompletionData(int index, string input)
{
if (!HandleInputStrings.Contains(input))
{
return(null);
}
var block = this._bracketSearcher.SearchCurrentBlock(this.Document, index - 1);
var bracket = string.Empty;
var inBlock = TokenType.StringBlock;
if (block != null)
{
bracket = this.Document.GetCharAt(block.OpenBracketOffset).ToString();
if (bracket == "{")
{
inBlock = TokenType.TagBlock;
}
else if (bracket == "[")
{
inBlock = TokenType.ArrayBlock;
}
}
var tokens = this.Text.Tokenize(TokenizeType.All);
var next = this.NextChar;
var prev = (this.CaretOffset > 1)
? this.Document.GetCharAt(this.CaretOffset - 2)
: '\0';
var before = tokens
.TakeWhile(x => x.Index < index)
.Where(x => !x.IsMatchType(TokenType.Blank));
var current = before.LastOrDefault();
var prevToken = before.SkipLast(1).LastOrDefault();
if (before.IsEmpty() ||
current.IsMatchType(TokenType.String, TokenType.Comment))
{
return(null);
}
if ("\r\n".Contains(input) &&
(Keyboard.Modifiers & ModifierKeys.Shift) != ModifierKeys.None)
{
return(null);
}
before = before.Where(x => !x.IsMatchType(TokenType.Comment));
using (var reader = new TokenReader(before, true))
{
try
{
#region コマンド
if (inBlock == TokenType.StringBlock)
{
switch (input.ToLower())
{
case "/":
return(Minecraft.CommandCompletion);
case "@":
return(Minecraft.SelectorCompletion);
case "_":
return((inBlock == TokenType.ArrayBlock && current.IsMatchType(TokenType.Literal))
? (current.Value.IndexOf("score_") == 0)
? (current.Value == "score_")
? this.ExtendedOptions.ScoreNames.ToCompletionData()
: new[] { new CompletionData("min") }
: null
: null);
}
/*
* if (!before.Any(x => x.IsMatchType(TokenType.Command)))
* {
* return null;
* }
*/
var command = reader.SkipGet(x => x.IsMatchType(TokenType.Command) || x.IsMatchLiteral("detect"));
if (command.IsEmpty())
{
return(null);
}
switch (input.ToLower())
{
case ":":
if (!command.IsEmpty())
{
if (reader.CheckAt(1, x => x.IsMatchLiteral("minecraft")))
{
goto case " ";
}
}
break;
case ",":
break;
case "\r\n":
case "\r":
case "\n":
case " ":
if (!command.IsEmpty() && inBlock == TokenType.StringBlock)
{
reader.Reverse();
var args = this.RemainTokensToArgs(reader);
var def = command.IsMatchLiteral("detect")
? Minecraft.Document.Root.Element("ExCommands").Elements("Command")
.Where(x => x.Attribute("name").Value == command.Value).SingleOrDefault()
: Minecraft.Document.Root.Element("Commands").Elements("Command")
.Where(x => "/" + x.Attribute("name").Value == command.Value).SingleOrDefault();
if (def == null)
{
return(null);
}
var elements = def.Elements("Arg");
var element = null as XElement;
foreach (var arg in args)
{
if (elements.IsEmpty())
{
return(null);
}
if (elements.Count() > 1)
{
Console.WriteLine(arg.Value);
if (elements.Any(x => x.Attribute("type").Value == "key" && arg.IsMatchLiteral(x.Attribute("key").Value)))
{
elements = elements.Where(x => arg.IsMatchLiteral(x.Attribute("key").Value)).Elements("Arg");
}
else if (elements.Any(x => x.Attributes().Any(a => a.Name == "default" && a.Value == "true")))
{
elements = elements.Where(x => x.Attributes().Any(a => a.Name == "default" && a.Value == "true")).Elements("Arg");
}
else
{
return(null);
}
}
else
{
elements = elements.Elements("Arg");
}
}
if (elements.Count() > 1)
{
if (elements.Any(x => x.Attributes().Any(a => a.Name == "default" && a.Value == "true")))
{
elements = elements.Where(x => x.Attributes().Any(a => a.Name == "default" && a.Value == "true"));
}
else
{
return(elements.Attributes()
.Where(x => x.Name == "key")
.Select(x => new CompletionData(x.Value)));
}
}
element = elements.SingleOrDefault();
if (element == null)
{
return(null);
}
var type = element.Attribute("type").Value;
switch (type)
{
case "key":
return(new[] { new CompletionData(element.Attribute("key").Value) });
case "keywords":
return(element.Elements("Word")
.Select(x =>
new
{
Name = x.Value.Trim(),
Tip = x.Attribute("desc") != null
? x.Attribute("desc").Value
: x.Element("Desc") != null
? x.Element("Desc").Value
: null
})
.Select(x => new CompletionData(x.Name, x.Tip)));
case "target":
return(this.ExtendedOptions.PlayerNames.ToCompletionData());
case "target_entity":
return(new[] { new CompletionData("@e") });
case "score":
return(this.ExtendedOptions.ScoreNames.ToCompletionData());
case "team":
return(this.ExtendedOptions.TeamNames.ToCompletionData());
case "item":
return(Minecraft.ItemCompletion);
case "block":
return(Minecraft.BlockCompletion);
case "entity":
return(Minecraft.EntityCompletion);
case "effect":
return(prevToken.IsMatchLiteral("minecraft")
? Minecraft.EffectCompletion
: Minecraft.EffectCompletion
.Concat(new[] { new CompletionData("clear", "すべてのエフェクトを消去します。") }));
case "enchant":
return(Minecraft.EnchantCompletion);
case "boolean":
return(Minecraft.BooleanCompletion);
case "color":
return(Minecraft.ColorCompletion);
case "command":
if (command.Value == "/execute" || command.Value == "detect")
{
return(new[] { new CompletionData("detect") }
.Concat(Minecraft.CommandCompletion));
}
return(Minecraft.CommandCompletion);
default:
return(null);
}
}
break;
}
}
#endregion
#region [ ... ]
if (inBlock == TokenType.ArrayBlock)
{
switch (input)
{
case "!":
if (reader.CheckNext(x => x.IsMatchType(TokenType.Equal)))
{
reader.MoveNext();
goto case "=";
}
return(null);
case "=":
if (reader.IsRemainToken)
{
var selector = before.TakeWhile(x => x.Index < block.OpenBracketOffset).LastOrDefault();
if (!selector.IsEmpty() && selector.IsMatchType(TokenType.TargetSelector))
{
reader.MoveNext();
if (reader.CheckNext(x => x.IsMatchLiteral("team")))
{
return(this.ExtendedOptions.TeamNames.ToCompletionData());
}
if (reader.CheckNext(x => x.IsMatchLiteral("name")))
{
return(this.ExtendedOptions.PlayerNames.ToCompletionData());
}
}
}
return(null);
}
}
#endregion
}
catch
{
return(null);
}
}
return(null);
}
// Push adds tr to the top (end) of the input stack. (Popping happens automatically.)
private static void Push(this ptr <Stack> _addr_s, TokenReader tr)
{
ref Stack s = ref _addr_s.val;
private List<DocumentToken> ReadReplacementBody(TokenReader<DocumentToken> reader, DocumentToken replacementKeyToken)
{
List<DocumentToken> replacementContents = new List<DocumentToken>();
int numOpenReplacements = 1;
while (reader.CanAdvance())
{
if (reader.Peek().TokenType == DocumentTokenType.BeginReplacementSegment)
{
numOpenReplacements++;
}
else if (reader.Peek().TokenType == DocumentTokenType.QuickTerminateReplacementSegment)
{
numOpenReplacements--;
if(numOpenReplacements == 0)
{
throw Unexpected(reader.Peek());
}
}
else if (reader.Peek().TokenType == DocumentTokenType.BeginTerminateReplacementSegment)
{
numOpenReplacements--;
if(numOpenReplacements == 0)
{
AdvanceAndExpectConstantType(reader, DocumentTokenType.BeginTerminateReplacementSegment);
AdvanceAndExpect(reader, DocumentTokenType.ReplacementKey, replacementKeyToken.Value, skipWhitespace: true);
AdvanceAndExpectConstantType(reader, DocumentTokenType.EndReplacementSegment);
break;
}
}
replacementContents.Add(reader.Advance());
}
if(numOpenReplacements != 0)
{
throw Unexpected("end of '" + replacementKeyToken.Value + "' replacement");
}
return replacementContents;
}
/// <summary>
/// 次に出現するトークンが特定の種類であることを確認し、読みます。<para />
/// 読めなかった場合はリワインドします。
/// </summary>
/// <param name="reader">トークン リーダー</param>
/// <param name="type">トークンの種類</param>
private static Token AssertNext(ref TokenReader reader, Token.TokenType type)
{
if (!reader.IsRemainToken)
reader.PushError("クエリが解析の途中で終了しました。" +
"ここには " + type.ToString() + " が存在しなければなりません。");
var ntoken = reader.Get();
if (ntoken.Type != type)
{
reader.PushError("不明な文字です: " + reader.LookAhead() + " (インデックス:" + reader.LookAhead().DebugIndex + ")" +
"ここには " + type.ToString() + " が存在しなければなりません。");
return new Token() { Type = type, Value = null, DebugIndex = -1 };
}
return ntoken;
}
/// <summary>
/// Takes in document replacement info and converts it into a document expression that represents an each loop.
/// </summary>
/// <param name="context">Context about the expression being parsed</param>
/// <returns>The parsed each expression</returns>
public IDocumentExpression CreateExpression(DocumentExpressionContext context)
{
if(context.Body.Count == 0)
{
throw new DocumentRenderException("Each tag has no body", context.ReplacementKeyToken);
}
TokenReader<DocumentToken> reader = new TokenReader<DocumentToken>(context.Parameters);
DocumentToken variableName;
if (reader.TryAdvance(out variableName, skipWhitespace: true) == false)
{
throw new DocumentRenderException("missing variable name in each tag", context.ReplacementKeyToken);
}
DocumentToken inToken;
if (reader.TryAdvance(out inToken, skipWhitespace: true) == false)
{
throw new DocumentRenderException("Expected 'in' after iteration variable", variableName);
}
if (inToken.Value != "in")
{
throw new DocumentRenderException("Expected 'in', got '" + inToken.Value + "'", inToken);
}
DocumentToken collectionExpressionToken;
if(reader.TryAdvance(out collectionExpressionToken, skipWhitespace: true) == false)
{
throw new DocumentRenderException("Expected collection expression after 'in' ", inToken);
}
DocumentToken unexpected;
if(reader.TryAdvance(out unexpected, skipWhitespace: true))
{
throw new DocumentRenderException("Unexpected parameter '" + unexpected.Value + "' after collection", unexpected);
}
return new EachExpression(variableName, collectionExpressionToken, context.Body);
}
private static ArgBodyTuple MakeArgBody(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeArgBody");
var token = reader.Get();
if (token.Type != Token.TokenType.String && token.Type != Token.TokenType.Literal)
{
reader.PushError("不明な文字です: " + reader.LookAhead() + " (インデックス:" + reader.LookAhead().DebugIndex + ")" +
"フィルタ引数は、単純な文字列かダブルクオートで括られた文字列のみが指定できます。");
}
var ret = new ArgBodyTuple() { Arg = token };
AssertNextAny(ref reader, false, Token.TokenType.Comma, Token.TokenType.CloseBracket,
Token.TokenType.ConcatenatorAnd, Token.TokenType.ConcatenatorOr);
return ret;
}
public void RegisterLexerTokenReader(TokenReader reader)
{
this.LexerTokenReaders.Add(reader);
}
private JReader(TokenReader tokenReader)
{
_tr = tokenReader;
_awaitingReadValue = false;
_delegate = null;
}
private IEnumerator <Parser> ParseFunction(RantCompiler compiler, CompileContext context, TokenReader reader,
Action <RST> actionCallback)
{
var functionName = reader.Read(R.Text, "acc-function-name");
var arguments = new List <RST>();
if (reader.PeekType() == R.Colon)
{
reader.ReadToken();
var iterator = ReadArguments(compiler, reader, arguments);
while (iterator.MoveNext())
{
yield return(iterator.Current);
}
compiler.SetNextActionCallback(actionCallback);
}
else
{
reader.Read(R.RightSquare);
}
RantFunctionSignature sig = null;
if (functionName.Value != null)
{
if (!RantFunctionRegistry.FunctionExists(functionName.Value))
{
compiler.SyntaxError(functionName, false, "err-compiler-nonexistent-function", functionName.Value, RantUtils.GetClosestFunctionName(functionName.Value));
yield break;
}
if ((sig = RantFunctionRegistry.GetFunction(functionName.Value, arguments.Count)) == null)
{
compiler.SyntaxError(functionName, false, "err-compiler-nonexistent-overload", functionName.Value, arguments.Count);
yield break;
}
actionCallback(new RstFunction(functionName.ToLocation(), sig, arguments));
}
}
private IEnumerator <Parser> ParseSubroutine(RantCompiler compiler, CompileContext context, TokenReader reader,
Action <RST> actionCallback)
{
// subroutine definition
if (reader.TakeLoose(R.LeftSquare, false))
{
if (reader.TakeLoose(R.Period))
{
compiler.HasModule = true;
}
var subroutineName = reader.ReadLoose(R.Text, "acc-subroutine-name");
var subroutine = new RstDefineSubroutine(subroutineName.ToLocation())
{
Parameters = new List <SubroutineParameter>(),
Name = subroutineName.Value
};
if (reader.PeekLooseToken().Type == R.Colon)
{
reader.ReadLooseToken();
do
{
var type = SubroutineParameterType.Greedy;
if (reader.TakeLoose(R.At))
{
type = SubroutineParameterType.Loose;
}
subroutine.Parameters.Add(new SubroutineParameter(reader.ReadLoose(R.Text, "acc-arg-name").Value, type));
} while (reader.TakeLoose(R.Semicolon, false));
}
reader.ReadLoose(R.RightSquare);
var bodyStart = reader.ReadLoose(R.Colon);
var actions = new List <RST>();
compiler.AddContext(CompileContext.SubroutineBody);
compiler.SetNextActionCallback(action => actions.Add(action));
yield return(Get <SequenceParser>());
compiler.SetNextActionCallback(actionCallback);
subroutine.Body = actions.Count == 1 ? actions[0] : new RstSequence(actions, bodyStart.ToLocation());
actionCallback(subroutine);
}
else
{
// subroutine call
var subroutineName = reader.Read(R.Text, "acc-subroutine-name");
var arguments = new List <RST>();
if (reader.PeekType() == R.Colon)
{
reader.ReadToken();
var iterator = ReadArguments(compiler, reader, arguments);
while (iterator.MoveNext())
{
yield return(iterator.Current);
}
compiler.SetNextActionCallback(actionCallback);
}
else
{
reader.Read(R.RightSquare);
}
var subroutine = new RstCallSubroutine(subroutineName.Value, subroutineName.ToLocation())
{
Arguments = arguments
};
actionCallback(subroutine);
}
}
/// <summary>
/// 次の一つのトークンがどれかに合致するか確認します。
/// </summary>
/// <param name="reader">リーダートークン</param>
/// <param name="isEnd">文末を受け入れるか</param>
/// <param name="tokens">可能なトークン集合</param>
private static void AssertNextAny(ref TokenReader reader, bool isEnd, params Token.TokenType[] tokens)
{
var descstr = "";
if (tokens.Count() == 1)
descstr = "ここには " + tokens.First().ToString() + " が存在しなければなりません。";
else
descstr = "ここには " + String.Join(", ", tokens.Select(s => s.ToString())) + " のうちいずれかが存在しなければなりません。";
if (!reader.IsRemainToken)
{
if (isEnd) return;
reader.PushError("クエリが解析の途中で終了しました。" + descstr);
}
if (tokens == null) return;
var lat = reader.LookAhead().Type;
if (!tokens.Any(f => f == lat))
{
reader.PushError("不明な文字です: " + reader.LookAhead() +
" (インデックス:" + reader.LookAhead().DebugIndex + ")" + descstr);
}
}
public override IEnumerator <Parser> Parse(RantCompiler compiler, CompileContext context, TokenReader reader,
Action <RST> actionCallback)
{
var nextType = reader.PeekType();
var tagStart = reader.PrevToken;
// replacer
switch (nextType)
{
case R.Regex:
{
var regex = reader.Read(R.Regex, "acc-replacer-regex");
var options = RegexOptions.Compiled | RegexOptions.ExplicitCapture;
if (reader.IsNext(R.RegexFlags))
{
var flagsToken = reader.ReadToken();
foreach (char flag in flagsToken.Value)
{
switch (flag)
{
case 'i':
options |= RegexOptions.IgnoreCase;
break;
case 'm':
options |= RegexOptions.Multiline;
break;
}
}
}
reader.Read(R.Colon);
var arguments = new List <RST>();
var iterator = ReadArguments(compiler, reader, arguments);
while (iterator.MoveNext())
{
yield return(iterator.Current);
}
compiler.SetNextActionCallback(actionCallback);
if (arguments.Count != 2)
{
compiler.SyntaxError(tagStart, reader.PrevToken, false, "err-compiler-replacer-argcount");
yield break;
}
actionCallback(new RstReplacer(regex.ToLocation(), new Regex(regex.Value, options), arguments[0], arguments[1]));
}
break;
case R.Dollar:
{
reader.ReadToken();
var e = ParseSubroutine(compiler, context, reader, actionCallback);
while (e.MoveNext())
{
yield return(e.Current);
}
}
break;
default:
{
var e = ParseFunction(compiler, context, reader, actionCallback);
while (e.MoveNext())
{
yield return(e.Current);
}
}
break;
}
}
private static ArgConcatrTuple MakeArgConcatr(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeArgConcatr");
ArgConcatrTuple ret = new ArgConcatrTuple();
if (TryLookAhead(ref reader, Token.TokenType.Comma))
{
ret.ArgBody = MakeArgBody(ref reader);
ret.ArgConcatr = MakeArgConcatr(ref reader);
}
AssertNextAny(ref reader, false, Token.TokenType.CloseBracket,
Token.TokenType.ConcatenatorAnd, Token.TokenType.ConcatenatorOr);
return ret;
}
public void OneTimeSetUp()
{
tokenReader = new TokenReader(MarkdownTokenUtilities.GetMarkdownTokenDescriptions());
renderer = MarkdownRenderUtilities.GetMarkdownRenderer();
}
private static ArgsTuple MakeArgs(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeArgs");
var ret = new ArgsTuple()
{
ArgBody = MakeArgBody(ref reader),
ArgConcatr = MakeArgConcatr(ref reader)
};
AssertNextAny(ref reader, false, Token.TokenType.CloseBracket,
Token.TokenType.ConcatenatorAnd, Token.TokenType.ConcatenatorOr);
return ret;
}
public static void Main()
{
var Nodes = new List <Node>();
var variables = new Dictionary <string, int>();
string source =
@"
a equals 10
b equals 15
decrement b
decrement b
decrement b
decrement b
decrement b
if a variable named a is not equal to a variable named b then
a equals b
end
write a to the console
";
Console.WriteLine(source);
bool inExpression = false;
NodeType currentMod = NodeType.NoOperation;
Queue <Node> BlockLevel = new Queue <Node>();
using (TokenReader tr = new TokenReader(source))
{
do
{
if (string.IsNullOrWhiteSpace(tr.Current()))
{
continue;
}
if (tr.EqualTo("if"))
{
BlockLevel.Enqueue(new Node(NodeType.JumpNotTrue));
inExpression = true;
}
else if (tr.EqualTo("then"))
{
Nodes.Add(BlockLevel.Peek());
inExpression = false;
}
else if (tr.EqualTo("end"))
{
BlockLevel.Dequeue().A = Nodes.Count;
Nodes.Add(new Node(NodeType.NoOperation));
}
else if (tr.EqualTo("increment"))
{
if (tr.MoveNext() && !tr.IsNumberLiteral())
{
Nodes.Add(new Node(NodeType.Increment,
GetVarIndex(tr.Current(), variables)));
}
else
{
throw new Exception("Expected a variable after increment");
}
}
else if (tr.EqualTo("decrement"))
{
if (tr.MoveNext() && !tr.IsNumberLiteral())
{
Nodes.Add(new Node(NodeType.Decrement,
GetVarIndex(tr.Current(), variables)));
}
else
{
throw new Exception("Expected a variable after increment");
}
}
else if (tr.EqualTo("write"))
{
if (tr.MoveNext())
{
int index = -1;
decimal value = 0;
if (tr.IsNumberLiteral())
{
value = tr.GetValue();
}
else
{
index = GetVarIndex(tr.Current(), variables);
}
if (tr.MoveNext())
{
if (tr.EqualTo("to", "the", "console"))
{
if (index == -1)
{
Nodes.Add(new Node(NodeType.LoadNumLiteralWriteConsole, value));
}
else
{
Nodes.Add(new Node(NodeType.LoadLocalWriteConsole, index));
}
tr.MoveNext(2);
}
}
else
{
throw new Exception("Expected what to write to, after value!");
}
}
else
{
throw new Exception("Expected a value to write");
}
}
else if (tr.EqualTo("is", "not", "equal", "to"))
{
// Nodes.Add(new Node(NodeType.NotEqual));
currentMod = NodeType.NotEqual;
tr.MoveNext(3);
}
else if (tr.EqualTo("is", "equal", "to"))
{
currentMod = NodeType.Equal;
//Nodes.Add(new Node(NodeType.Equal));
tr.MoveNext(2);
}
else if (tr.EqualTo("a", "variable", "named"))
{
if (tr.MoveNext(3))
{
Nodes.Add(new Node(NodeType.LoadLocal,
GetVarIndex(tr.Current(), variables)));
if (currentMod != NodeType.NoOperation)
{
Nodes.Add(new Node(currentMod));
currentMod = NodeType.NoOperation;
}
}
else
{
throw new Exception("Expected name after a variable named");
}
}
else
{
if (inExpression)
{
if (tr.IsNumberLiteral())
{
Nodes.Add(new Node(NodeType.LoadNumLiteral, tr.GetValue()));
}
else
{
Nodes.Add(new Node(NodeType.LoadLocal,
GetVarIndex(tr.Current(), variables)));
}
if (currentMod != NodeType.NoOperation)
{
Nodes.Add(new Node(currentMod));
currentMod = NodeType.NoOperation;
}
}
else
{
int index = GetVarIndex(tr.Current(), variables);
if (tr.MoveNext() && tr.EqualTo("equals"))
{
tr.MoveNext();
if (tr.IsNumberLiteral())
{
Nodes.Add(new Node(NodeType.LoadNumLiteralAndStoreLocal, index, tr.GetValue()));
}
else
{
Nodes.Add(new Node(NodeType.LoadLocalAndStoreLocal, index,
GetVarIndex(tr.Current(), variables)));
}
}
else
{
throw new Exception("Expected something after the variable");
}
}
}
} while (tr.MoveNext());
ExecuteNodes(Nodes, variables.Count);
}
}
private static ConcatenatorTuple MakeConcatenator(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeConcatenator");
ConcatenatorTuple ret = new ConcatenatorTuple();
if (TryLookAhead(ref reader, Token.TokenType.ConcatenatorAnd))
{
ret.IsConcatenateAnd = true;
ret.Expression = MakeExpression(ref reader);
ret.Concatenator = MakeConcatenator(ref reader);
}
else if (TryLookAhead(ref reader, Token.TokenType.ConcatenatorOr))
{
ret.IsConcatenateAnd = false;
ret.Expression = MakeExpression(ref reader);
ret.Concatenator = MakeConcatenator(ref reader);
}
else
{
ret = new ConcatenatorTuple();
}
AssertNextAny(ref reader, false, Token.TokenType.CloseBracket);
return ret;
}
public override IEnumerator <Parser> Parse(RantCompiler compiler, CompileContext context, TokenReader reader, Action <RantAction> actionCallback)
{
throw new NotImplementedException();
}
private static FilterTuple MakeFilter(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeFilter");
var ret = new FilterTuple()
{
Name = AssertNext(ref reader, Token.TokenType.Literal),
FilterAttr = MakeFilterAttr(ref reader)
};
AssertNextAny(ref reader, false, Token.TokenType.CloseBracket,
Token.TokenType.ConcatenatorAnd, Token.TokenType.ConcatenatorOr);
return ret;
}
private static List <Menu> GenerateMenu()
{
List <Menu> menu = new List <Menu>();
menu.Add(new Menu("Enter a new Shape",
delegate(ShapesContainer container)
{
Console.WriteLine("Enter a new Shape");
string input = Console.ReadLine();
try
{
Console.WriteLine(container.AddShape(TokenReader.ReadToken(input)));
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
}
}));
menu.Add(new Menu("Load shapes from file", delegate(ShapesContainer container) {
Console.WriteLine("Enter the file complete path");
string input = Console.ReadLine();
if (!System.IO.File.Exists(input))
{
Console.WriteLine(String.Format("The file {0} does not exists.", input));
return;
}
string[] lines = System.IO.File.ReadAllLines(input);
foreach (string shape in lines)
{
if (shape.Trim() != "")
{
try
{
Console.WriteLine(container.AddShape(TokenReader.ReadToken(shape.Trim())));
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
}
}
}
}));
menu.Add(new Menu("Delete a Shape",
delegate(ShapesContainer container)
{
Console.WriteLine("Enter the shape id");
string input = Console.ReadLine();
try
{
Console.WriteLine(container.deleteShape(TokenReader.ReadId(input)));
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
}
}));
menu.Add(new Menu("Find shapes that includes point",
delegate(ShapesContainer container)
{
Console.WriteLine("Find shapes that includes point x y");
try
{
string input = Console.ReadLine();
var point = TokenReader.ReadPoint(input);
foreach (string sh in container.FindInPoint(point[0], point[1]))
{
Console.WriteLine(sh);
}
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
}
}));
menu.Add(new Menu("List all the shapes",
delegate(ShapesContainer container) {
Console.WriteLine("List all the shapes");
foreach (string sh in container.GetShapeList())
{
Console.WriteLine(sh);
}
}));
menu.Add(new Menu("Help", delegate(ShapesContainer container) {
Console.WriteLine("Help for shapes manager: ");
Console.WriteLine("--------------------------------");
Console.WriteLine("--------------------------------");
Console.WriteLine("- Enter a new Shap: ");
Console.WriteLine(" This command allows you to enter a new shape with the format: shapename pointx pointy [list of points] ");
Console.WriteLine(" Each shape change lightly the params they need to be created ");
Console.WriteLine(" Circle: circle x y radius ");
Console.WriteLine(" Square: square x y side ");
Console.WriteLine(" Rectangle: rectangle x y side1 side2 ");
Console.WriteLine(" Triangle: triangle x_vertice1 y_vertice1 x_vertice2 y_vertice2 x_vertice3 y_vertice3 ");
Console.WriteLine(" Donut: donut x y radius1 radius2 ");
Console.WriteLine(" Ellipse: ellipse x y radiusMin radiusMax ");
Console.WriteLine(" ** If the format or number of the params are incorrect, the application will show an error. ");
Console.WriteLine("--------------------------------");
Console.WriteLine("- Load shapes from file: ");
Console.WriteLine(" This will load a list of shapes (with the correct format) from a file ");
Console.WriteLine(" ** If file does not exists, the application will show an error. ");
Console.WriteLine(" ** If one of the shapes have a wrong format, the application will show the line error and will continue with the rest shapes. ");
Console.WriteLine("--------------------------------");
Console.WriteLine("- Delete a Shape: ");
Console.WriteLine(" Deletes a shape with the id specified ");
Console.WriteLine(" The id should be an integer number ");
Console.WriteLine(" ** If the shape with the specified id does not exists, the application will show an error. ");
Console.WriteLine("--------------------------------");
Console.WriteLine("- Find shapes that includes point: ");
Console.WriteLine(" This will scan all the shapes and list the ones that will contain the point within its area ");
Console.WriteLine(" The point form with 2 floating numbers separate by one space ");
Console.WriteLine(" ** If the point is not format correctly the application will show an error ");
Console.WriteLine(" ** If non shape is containing the point, the application will show 0 results ");
Console.WriteLine("--------------------------------");
Console.WriteLine("- List all the shapes: ");
Console.WriteLine(" This option will print all the shapes ");
Console.WriteLine(" ** If there are no shapes, no error will be printed ");
Console.WriteLine("--------------------------------");
Console.WriteLine("- Help: ");
Console.WriteLine(" Shows this help ");
Console.WriteLine("--------------------------------");
Console.WriteLine("- Exit: ");
Console.WriteLine(" Exits the application ");
Console.WriteLine(" ** The application does not persist any data, all the shapes that where added will be lost when exits the application ");
Console.WriteLine("--------------------------------");
Console.WriteLine("--------------------------------");
}));
menu.Add(new Menu("Exit", delegate(ShapesContainer container) {
Environment.Exit(0);
}));
return(menu);
}
private static InnerClusterTuple MakeInnerCluster(ref TokenReader reader)
{
System.Diagnostics.Debug.WriteLine("MakeInnerCluster");
// Follow -> )
if (TryLookAhead(ref reader, Token.TokenType.CloseBracket, false))
return new InnerClusterTuple();
else
{
var rct = new InnerClusterTuple()
{
Expression = MakeExpression(ref reader),
Concatenator = MakeConcatenator(ref reader)
};
AssertNextAny(ref reader, false, Token.TokenType.CloseBracket);
return rct;
}
}
public SimpleQueryParser(string str)
{
_tokenReader = new TokenReader(str);
}
static SyntaxTuple MakeTuples(IEnumerable<Token> tokens)
{
var reader = new TokenReader(tokens);
RootTuple roots = null;
try
{
roots = MakeRoot(ref reader);
}
catch (ArgumentException e)
{
throw new ArgumentException(e.Message + Environment.NewLine +
"他のエラー:" + String.Join(Environment.NewLine, reader.Errors()), e);
}
if (reader.IsContainErrors)
{
throw new ArgumentException("シンタックス エラー:" + String.Join(Environment.NewLine, reader.Errors()));
}
if (reader.IsRemainToken)
{
throw new ArgumentException("閉じ括弧が多すぎる可能性があります。(クエリは途中で解析を完了しました) (next:@" + reader.Get().DebugIndex + ")");
}
return roots;
}
public void Test_token_reader()
{
TokenReader reader = new TokenReader();
var tokens = reader.Read("iif(Remark == abs,2,1)");
}
// NOTE: one thing i don't really like is having to create a completely new compiler for each pattern you run. // figure out a way to create one compiler at the start, and use that for all patterns? private bool MatchingCompilerAndReader(RantCompiler compiler, TokenReader reader) => this.compiler == compiler && this.reader == reader;
public TConceptInfo QuickParse(string dsl, Stack <IConceptInfo> context)
{
tokenReader = GenericParserTest.TestTokenReader(dsl);
return((TConceptInfo)Parse(tokenReader, context, out var warnings).Value);
}
public static void SetCompilerAndReader(RantCompiler c, TokenReader r)
{
sCompiler = c;
sReader = r;
}
//--------------------------------------------------------Misc Methods:---------------------------------------------------------------\\
#region --Misc Methods (Public)--
#endregion
#region --Misc Methods (Private)--
private void LoadMapApiKey()
{
string token = TokenReader.LoadTokenFromFile(TokenReader.MAP_TOKEN_PATH);
canteens_map.MapServiceToken = token;
}
//================================================================
// Constructor uses given ICharInputStream
//
internal Tokenizer (BinaryReader input)
{
LineNo = 1 ;
_fintag = false ;
_fincstr = false ;
_cstr = null ;
_input = new TokenReader(input) ;
_sbarray = new char[_sbmaxsize];
_sbindex = 0;
}
private void ParseReplacement(TokenReader<DocumentToken> reader, List<IDocumentExpression> ret)
{
var openToken = AdvanceAndExpectConstantType(reader, DocumentTokenType.BeginReplacementSegment);
var replacementKeyToken = AdvanceAndExpect(reader, DocumentTokenType.ReplacementKey, "replacement key", skipWhitespace: true);
List<DocumentToken> parameters = new List<DocumentToken>();
List<DocumentToken> body = new List<DocumentToken>();
while (reader.CanAdvance(skipWhitespace: true) && reader.Peek(skipWhitespace: true).TokenType == DocumentTokenType.ReplacementParameter)
{
var paramToken = reader.Advance(skipWhitespace: true);
parameters.Add(paramToken);
}
DocumentToken closeReplacementToken;
if(reader.TryAdvance(out closeReplacementToken, skipWhitespace: true) == false)
{
throw Unexpected(string.Format("'{0}' or '{1}'", DocumentToken.GetTokenTypeValue(DocumentTokenType.EndReplacementSegment), DocumentToken.GetTokenTypeValue(DocumentTokenType.QuickTerminateReplacementSegment)));
}
if (closeReplacementToken.TokenType == DocumentTokenType.EndReplacementSegment)
{
body.AddRange(ReadReplacementBody(reader, replacementKeyToken));
}
else if (closeReplacementToken.TokenType == DocumentTokenType.QuickTerminateReplacementSegment)
{
// do nothing, there is no body when the quick termination replacement segment is used
}
else
{
throw Unexpected(string.Format("'{0}' or '{1}'", DocumentToken.GetTokenTypeValue(DocumentTokenType.EndReplacementSegment), DocumentToken.GetTokenTypeValue(DocumentTokenType.QuickTerminateReplacementSegment)), closeReplacementToken);
}
IDocumentExpressionProvider provider;
if (this.expressionProviders.TryGetValue(replacementKeyToken.Value, out provider) == false)
{
provider = new EvalExpressionProvider();
}
var context = new DocumentExpressionContext
{
OpenToken = openToken,
CloseToken = closeReplacementToken,
Parameters = parameters.AsReadOnly(),
Body = body.AsReadOnly(),
ReplacementKeyToken = replacementKeyToken,
};
var expression = provider.CreateExpression(context);
ret.Add(expression);
}
/// <summary>
/// Creates an eval expression given template replacement info
/// </summary>
/// <param name="context">The context that contains information about the document being rendered</param>
/// <returns></returns>
public IDocumentExpression CreateExpression(DocumentExpressionContext context)
{
if (context.Body.Count > 0)
{
throw new DocumentRenderException("eval tags can't have a body", context.ReplacementKeyToken);
}
TokenReader<DocumentToken> reader = new TokenReader<DocumentToken>(context.Parameters);
DocumentToken fgToken, bgToken;
var ret = new EvalExpression(context.ReplacementKeyToken);
if(reader.TryAdvance(out fgToken, skipWhitespace: true) == false)
{
return ret;
}
else
{
ret.ForegroundColorToken = fgToken;
}
if (reader.TryAdvance(out bgToken, skipWhitespace: true) == false)
{
return ret;
}
else
{
ret.BackgroundColorToken = bgToken;
}
return ret;
}
public TemplateCodeGen(String TemplateString, TemplateFactory TemplateFactory = null)
{
this.TemplateFactory = TemplateFactory;
this.Tokens = new TokenReader(TemplateTokenizer.Tokenize(new TokenizerStringReader(TemplateString)));
}
