private Scanner get_scanner()
{
if (scanner != null)
{
return(scanner);
}
var sg = new ScannerGenerator();
sg.PushRule("", @"[\r\n ]"); // Skip characters
sg.PushRule("open", @"\<");
sg.PushRule("empty-open", @"\<\/");
sg.PushRule("close", @"\>");
sg.PushRule("empty-close", @"\/\>");
sg.PushRule("annoate-open", @"\<!\-\-");
sg.PushRule("annoate-close", @"\-\-\>");
sg.PushRule("annoate2-open", @"\<!");
sg.PushRule("tag-name", @"[a-zA-Z0-9]+");
sg.PushRule("attr-name", @"""([^\\""]|\\"")*""");
sg.Generate();
Console.Console.Instance.WriteLine(sg.PrintDiagram());
return(scanner = sg.CreateScannerInstance());
}
private void bLG_Click(object sender, EventArgs e)
{
var sg = new ScannerGenerator();
try
{
foreach (var line in rtbLLD.Lines)
{
sg.PushRule(line.Split(new[] { "=>" }, StringSplitOptions.None)[1].Replace("\"", "").Trim(), line.Split(new[] { "=>" }, StringSplitOptions.None)[0].Trim());
}
sg.Generate();
scanner = sg.CreateScannerInstance();
rtbLS.AppendText("New scanner instance generated!\r\n" + sg.PrintDiagram());
}
catch (Exception ex)
{
rtbLS.Text = $"Scanner build error!\r\n{ex.Message}\r\n{ex.StackTrace}";
}
}
public static void Main(string[] args)
{
var sg = new ScannerGenerator();
sg.PushRule("", "[\\r\\n ]");
sg.PushRule("if", "if");
sg.PushRule("for", "for");
sg.PushRule("else", "else");
sg.PushRule("id", "[a-z][a-z0-9]*");
sg.PushRule("num", "[0-9]+");
sg.Generate();
sg.PrintDiagram();
var scanner = sg.CreateScannerInstance();
scanner.AllocateTarget("+/a1321 if else 0415abse+9999");
while (scanner.Valid())
{
var ss = scanner.Next();
Console.WriteLine($"{ss.Item1}, {ss.Item2}");
}
}
private Scanner get_scanner()
{
if (scanner != null)
{
return(scanner);
}
var sg = new ScannerGenerator();
sg.PushRule("", @"[\r\n ]");
sg.PushRule("op_open", @"\(");
sg.PushRule("op_close", @"\)");
sg.PushRule("pp_open", @"\[");
sg.PushRule("pp_close", @"\]");
sg.PushRule("equal", @"\=");
sg.PushRule("plus", @"\+");
sg.PushRule("minus", @"\-");
sg.PushRule("multiple", @"\*");
sg.PushRule("divide", @"\/");
sg.PushRule("to", "to");
sg.PushRule("scolon", ":");
sg.PushRule("comma", ",");
sg.PushRule("loop", "loop");
sg.PushRule("foreach", "foreach");
sg.PushRule("if", "if");
sg.PushRule("else", "else");
sg.PushRule("id", @"[_$a-zA-Z][_$a-zA-Z0-9]*");
sg.PushRule("num", @"[0-9]+(\.[0-9]+)?([Ee][\+\-]?[0-9]+)?");
sg.PushRule("str", @"""[_$a-zA-Z0-9\/\?:,\[\]\\#\=&\+\-\*\|\(\)\<\>\.{}! ]*""");
sg.Generate();
Console.Console.Instance.WriteLine(sg.PrintDiagram());
return(scanner = sg.CreateScannerInstance());
}
static Scanner CreateJSonLexer()
{
var sg = new ScannerGenerator();
sg.PushRule("", @"[\t\r\n ]");
sg.PushRule("comma", "\\,");
sg.PushRule("v_pair", "\\:");
sg.PushRule("object_starts", "\\{");
sg.PushRule("object_ends", "\\}");
sg.PushRule("array_starts", "\\[");
sg.PushRule("array_ends", "\\]");
sg.PushRule("v_true", "true");
sg.PushRule("v_false", "false");
sg.PushRule("v_null", "null");
sg.PushRule("v_string", @"""([^""\\]|\\.)*""");
sg.PushRule("v_number", @"-?[0-9]+(\.[0-9]+)?([Ee][\+\-]?[0-9]+)?");
sg.Generate();
#if false
Console.WriteLine(sg.PrintDiagram());
#endif
return(sg.CreateScannerInstance());
}
/// <summary>
/// Create JavaScript Scanner
/// </summary>
private static void create_scanner()
{
var gen = new ScannerGenerator();
gen.PushRule("", @"[\r\n ]");
//gen.PushRule("", @"");
gen.PushRule("Class", "class");
gen.PushRule("Break", "break");
gen.PushRule("Do", "do");
gen.PushRule("Instanceof", "instanceof");
gen.PushRule("Typeof", "typeof");
gen.PushRule("Case", "case");
gen.PushRule("Else", "else");
gen.PushRule("New", "new");
gen.PushRule("Var", "var");
gen.PushRule("Catch", "catch");
gen.PushRule("Finally", "finally");
gen.PushRule("Return", "return");
gen.PushRule("Void", "void");
gen.PushRule("Continue", "continue");
gen.PushRule("For", "for");
gen.PushRule("Switch", "switch");
gen.PushRule("While", "while");
gen.PushRule("Debugger", "debugger");
gen.PushRule("Function", "function");
gen.PushRule("This", "this");
gen.PushRule("With", "with");
gen.PushRule("Default", "default");
gen.PushRule("If", "if");
gen.PushRule("Throw", "throw");
gen.PushRule("Delete", "delete");
gen.PushRule("In", "in");
gen.PushRule("Try", "try");
gen.PushRule("Enum", "enum");
gen.PushRule("Extends", "extends");
gen.PushRule("Super", "super");
gen.PushRule("Const", "const");
gen.PushRule("Export", "export");
gen.PushRule("Import", "import");
gen.PushRule("Implements", "implements");
gen.PushRule("Let", "let");
gen.PushRule("Private", "private");
gen.PushRule("Public", "public");
gen.PushRule("Interface", "interface");
gen.PushRule("Package", "package");
gen.PushRule("Protected", "protected");
gen.PushRule("Static", "static");
gen.PushRule("Yield", "yield");
//gen.PushRule("SemiColon", @"SemiColon");
gen.PushRule("{", @"\{");
gen.PushRule("}", @"\}");
gen.PushRule(",", @"\,");
gen.PushRule("(", @"\(");
gen.PushRule(")", @"\)");
gen.PushRule(";", @"\;");
gen.PushRule("*", @"\*");
gen.PushRule("[", @"\[");
gen.PushRule("]", @"\]");
gen.PushRule(".", @"\.");
gen.PushRule("++", @"\+\+");
gen.PushRule("--", @"\-\-");
gen.PushRule("+", @"\+");
gen.PushRule("-", @"\-");
gen.PushRule("~", @"\~");
gen.PushRule("!", @"\!");
gen.PushRule("/", @"\/");
gen.PushRule("%", @"\%");
gen.PushRule("<<", @"\<\<");
gen.PushRule(">>", @"\>\>");
gen.PushRule(">>>", @"\>\>\>");
gen.PushRule("<", @"\<");
gen.PushRule(">", @"\>");
gen.PushRule("<=", @"\<\=");
gen.PushRule(">=", @"\>\=");
gen.PushRule("==", @"\=\=");
gen.PushRule("===", @"\=\=\=");
gen.PushRule("!==", @"\!\=\=");
gen.PushRule("&", @"\&");
gen.PushRule("^", @"\^");
gen.PushRule("|", @"\|");
gen.PushRule("&&", @"\&\&");
gen.PushRule("||", @"\|\|");
gen.PushRule("?", @"\?");
gen.PushRule("=>", @"\=\>");
gen.PushRule("*=", @"\*\=");
gen.PushRule("/=", @"\/\=");
gen.PushRule("%=", @"\%\=");
gen.PushRule("+=", @"\+\=");
gen.PushRule("-=", @"\-\=");
gen.PushRule("<<=", @"\<\<\=");
gen.PushRule(">>=", @"\>\>\=");
gen.PushRule("&=", @"\&\=");
gen.PushRule("^=", @"\^\=");
gen.PushRule("|=", @"\|\=");
gen.PushRule("StringLiteral", @"""([^""\\]|\\.)*""");
gen.PushRule("DecimalLiteral", @"[0-9]+(\.[0-9]+)?([Ee][\+\-]?[0-9]+)?");
gen.PushRule("BinaryIntegerLiteral", @"[0-9]+b");
gen.PushRule("HexIntegerLiteral", @"0x[0-9]+");
gen.PushRule("OctalIntegerLiteral", @"0[oO][0-9]+");
gen.PushRule("BooleanLiteral", @"true|false");
gen.PushRule("Identifier", @"[_$a-zA-Z][_$a-zA-Z0-9]*");
gen.Generate();
Console.Console.Instance.WriteLine(gen.PrintDiagram());
scanner = gen.CreateScannerInstance();
}
private Scanner get_scanner()
{
if (scanner != null)
{
return(scanner);
}
var sg = new ScannerGenerator();
sg.PushRule("", @"[\r\n ]"); // Skip characters
sg.PushRule("loop", "loop");
sg.PushRule("op_open", @"\(");
sg.PushRule("op_close", @"\)");
sg.PushRule("pp_open", @"\[");
sg.PushRule("pp_close", @"\]");
sg.PushRule("equal", @"\=");
sg.PushRule("to", "to");
sg.PushRule("scolon", ":");
sg.PushRule("comma", ",");
sg.PushRule("foreach", "foreach");
sg.PushRule("if", "if");
sg.PushRule("else", "else");
sg.PushRule("name", @"[_$a-zA-Z][_$a-zA-Z0-9]*");
//sg.PushRule("const", @"[\-\+]?[0-9]+(\.[0-9]+)?([Ee][\+\-]?[0-9]+)?|""[_$a-zA-Z0-9\/\?:,\[\]\\#\=&\+\-\*\|\(\)\<\>\.{}! ]*""");
//sg.PushRule("const", @"[\-\+]?[0-9]+(\.[0-9]+)?([Ee][\+\-]?[0-9]+)?|""([^\\""]|\\"")*""");
sg.PushRule("const", @"[\-\+]?[0-9]+(\.[0-9]+)?([Ee][\+\-]?[0-9]+)?|""([^""\\]|\\.)*""");
sg.Generate();
Console.Console.Instance.WriteLine(sg.PrintDiagram());
return(scanner = sg.CreateScannerInstance());
}
/// <summary>
///
/// </summary>
static void ProcessTest(string[] args)
{
switch (args[0].ToInt32())
{
case 1:
var c2c = new CALtoCS();
c2c.Compile(File.ReadAllLines("script/danbooru-pages.srcal"));
break;
case 2:
var cal = new ESRCAL();
cal.Compile(new[] { "5+4-(-4*(2-4)*2)/3+-(-(2*2+3)-2)*(3+1)" });
//cal.Compile(new[] { "(2*2+3)" });
break;
case 3:
var x = new HTMLParser();
break;
case 4:
var p = JSParserGenerator.Parser;
break;
case 5:
var s = JSScannerGenerator.Scanner;
break;
case 6:
var sg = new ScannerGenerator();
sg.PushRule("", @"[\t\r\n ]");
sg.PushRule("comma", "\\,");
sg.PushRule("v_pair", "\\:");
sg.PushRule("object_starts", "\\{");
sg.PushRule("object_ends", "\\}");
sg.PushRule("array_starts", "\\[");
sg.PushRule("array_ends", "\\]");
sg.PushRule("v_true", "true");
sg.PushRule("v_false", "false");
sg.PushRule("v_null", "null");
var vv = new List <StringBuilder>(5);
vv.Add(new StringBuilder()); // 6
vv.Add(new StringBuilder()); // 5
vv.Add(new StringBuilder()); // 4
vv.Add(new StringBuilder()); // 3
vv.Add(new StringBuilder()); // 2
for (int cc = 0xc0; cc < 0xff; cc++)
{
if ((cc & 0xfc) == 0xfc)
{
vv[0].Append("\\x" + cc.ToString("X2"));
}
else if ((cc & 0xf8) == 0xf8)
{
vv[1].Append("\\x" + cc.ToString("X2"));
}
else if ((cc & 0xf0) == 0xf0)
{
vv[2].Append("\\x" + cc.ToString("X2"));
}
else if ((cc & 0xe0) == 0xe0)
{
vv[3].Append("\\x" + cc.ToString("X2"));
}
else if ((cc & 0xc0) == 0xc0)
{
vv[4].Append("\\x" + cc.ToString("X2"));
}
}
var bb = new StringBuilder();
for (int i = 0, j = 5; i < 5; i++, j--)
{
bb.Append("[" + vv[i].ToString() + "]" + new string('.', j));
if (i != 4)
{
bb.Append("|");
}
}
//SimpleRegex sr = new SimpleRegex();
//sr.MakeNFA($"\"({bb.ToString()}|[^\\\"]|\\\")*\"");
//sr.OptimizeNFA();
//sr.NFAtoDFA();
//sr.MinimizeDFA();
//Console.Instance.WriteLine(sr.PrintDiagram());
//sg.PushRule("v_string", $"\"({bb.ToString()}|[^\\\"]|\\\")*\"");
sg.PushRule("v_string", $"\"([^\\\"]|\\\")*\"");
sg.PushRule("v_number", @"-?[0-9]+(\.[0-9]+)?([Ee][\+\-]?[0-9]+)?");
sg.Generate();
Console.Instance.WriteLine(sg.PrintDiagram());
var s3 = sg.CreateScannerInstance();
Console.Instance.WriteLine(s3.ToCSCode("json_lexer"));
var gen = new ParserGenerator();
var JSON = gen.CreateNewProduction("JSON", false);
var ARRAY = gen.CreateNewProduction("ARRAY", false);
var OBJECT = gen.CreateNewProduction("OBJECT", false);
var MEMBERS = gen.CreateNewProduction("MEMBERS", false);
var PAIR = gen.CreateNewProduction("PAIR", false);
var ELEMENTS = gen.CreateNewProduction("ELEMENTS", false);
var VALUE = gen.CreateNewProduction("VALUE", false);
var object_starts = gen.CreateNewProduction("object_starts");
var object_ends = gen.CreateNewProduction("object_ends");
var comma = gen.CreateNewProduction("comma");
var v_pair = gen.CreateNewProduction("v_pair");
var array_starts = gen.CreateNewProduction("array_starts");
var array_ends = gen.CreateNewProduction("array_ends");
var v_true = gen.CreateNewProduction("v_true");
var v_false = gen.CreateNewProduction("v_false");
var v_null = gen.CreateNewProduction("v_null");
var v_string = gen.CreateNewProduction("v_string");
var v_number = gen.CreateNewProduction("v_number");
JSON |= OBJECT;
JSON |= ARRAY;
OBJECT |= object_starts + object_ends;
OBJECT |= object_starts + MEMBERS + object_ends;
MEMBERS |= PAIR;
MEMBERS |= PAIR + comma + MEMBERS;
PAIR |= v_string + v_pair + VALUE;
ARRAY |= array_starts + array_ends;
ARRAY |= array_starts + ELEMENTS + array_ends;
ELEMENTS |= VALUE;
ELEMENTS |= VALUE + comma + ELEMENTS;
VALUE |= v_string;
VALUE |= v_number;
VALUE |= OBJECT;
VALUE |= ARRAY;
VALUE |= v_true;
VALUE |= v_false;
VALUE |= v_null;
gen.PushStarts(JSON);
gen.PrintProductionRules();
gen.GenerateLALR2();
gen.PrintStates();
gen.PrintTable();
Console.Instance.WriteLine(gen.GlobalPrinter.ToString());
Console.Instance.WriteLine(gen.CreateExtendedShiftReduceParserInstance().ToCSCode("json_parser"));
break;
}
}
public static void Main(string[] args)
{
var gen = new ParserGenerator();
// Non-Terminals
var exp = gen.CreateNewProduction("exp", false);
var term = gen.CreateNewProduction("term", false);
var factor = gen.CreateNewProduction("factor", false);
var func = gen.CreateNewProduction("func", false);
var arguments = gen.CreateNewProduction("args", false);
var args_left = gen.CreateNewProduction("args_left", false);
// Terminals
var plus = gen.CreateNewProduction("plus"); // +
var minus = gen.CreateNewProduction("minus"); // -
var multiple = gen.CreateNewProduction("multiple"); // *
var divide = gen.CreateNewProduction("divide"); // /
var id = gen.CreateNewProduction("id"); // [_$a-zA-Z][_$a-zA-Z0-9]*
var op_open = gen.CreateNewProduction("op_open"); // (
var op_close = gen.CreateNewProduction("op_close"); // )
var num = gen.CreateNewProduction("num"); // [0-9]+
var split = gen.CreateNewProduction("split"); // ,
exp |= exp + plus + term;
exp |= exp + minus + term;
exp |= term;
term |= term + multiple + factor;
term |= term + divide + factor;
term |= factor;
factor |= op_open + exp + op_close;
factor |= num;
factor |= id;
factor |= func;
func |= id + op_open + arguments + op_close;
arguments |= args_left + id;
arguments |= ParserGenerator.EmptyString;
args_left |= args_left + id + split;
args_left |= ParserGenerator.EmptyString;
gen.PushStarts(exp);
gen.Generate();
gen.PrintStates();
var slr = gen.CreateShiftReduceParserInstance();
// 2*4+5$
Action <string, string> insert = (string x, string y) =>
{
slr.Insert(x, y);
while (slr.Reduce())
{
var l = slr.LatestReduce();
Console.Write(l.Produnction.PadLeft(8) + " => ");
Console.WriteLine(string.Join(" ", l.Childs.Select(z => z.Produnction)));
Console.Write(l.Produnction.PadLeft(8) + " => ");
Console.WriteLine(string.Join(" ", l.Childs.Select(z => z.Contents)));
slr.Insert(x, y);
}
};
var sg2 = new ScannerGenerator();
sg2.PushRule("", "[\\r\\n ]");
sg2.PushRule("plus", "\\+");
sg2.PushRule("minus", "-");
sg2.PushRule("multiple", "\\*");
sg2.PushRule("divide", "\\/");
sg2.PushRule("op_open", "\\(");
sg2.PushRule("op_close", "\\)");
sg2.PushRule("split", ",");
sg2.PushRule("id", "[a-z][a-z0-9]*");
sg2.PushRule("num", "[0-9]+");
sg2.Generate();
sg2.CreateScannerInstance();
var scanner2 = sg2.CreateScannerInstance();
scanner2.AllocateTarget("2+6*(6+4*7-2)+sin(a,b)+cos()*pi");
while (scanner2.Valid())
{
var ss = scanner2.Next();
insert(ss.Item1, ss.Item2);
}
insert("$", "$");
}