public static void showtree(grammar g, node n)
{
int i;
if (n == null)
{
return;
}
if (ISNONTERMINAL(TYPE(n)))
{
for (i = 0; i < NCH(n); i++)
{
showtree(g, CHILD(n, i));
}
}
else if (ISTERMINAL(TYPE(n)))
{
printf("%s", _PyParser_TokenNames[TYPE(n)]);
if (TYPE(n) == NUMBER || TYPE(n) == NAME)
{
printf("(%s)", STR(n));
}
printf(" ");
}
else
{
printf("? ");
}
}
public static parser_state PyParser_New(grammar g, int start)
{
parser_state ps;
if (0 == g.g_accel)
{
PyGrammar_AddAccelerators(g);
}
ps = PyMem_NEW_parser_state(1);
if (ps == null)
{
return(null);
}
ps.p_grammar = g;
ps.p_generators = 0;
ps.p_tree = PyNode_New(start);
if (ps.p_tree == null)
{
PyMem_DEL(ref ps);
return(null);
}
s_reset(ps.p_stack);
s_push(ps.p_stack, PyGrammar_FindDFA(g, start), ps.p_tree);
return(ps);
}
public static void dumptree(grammar g, node n)
{
int i;
if (n == null)
{
printf("NIL");
}
else
{
label l = new label();
l.lb_type = TYPE(n);
l.lb_str = STR(n);
printf("%s", PyGrammar_LabelRepr(l));
if (ISNONTERMINAL(TYPE(n)))
{
printf("(");
for (i = 0; i < NCH(n); i++)
{
if (i > 0)
{
printf(",");
}
dumptree(g, CHILD(n, i));
}
printf(")");
}
}
}
//20170403
//#include "python.h"
//#include "pgenheaders.h"
//#include "grammar.h"
//#include "token.h"
public static dfa PyGrammar_FindDFA(grammar g, int type)
{
dfa d;
d = g.g_dfa[type - NT_OFFSET];
assert(d.d_type == type);
return(d);
}
private static void fixdfa(grammar g, dfa d)
{
statePtr s;
int j;
s = new statePtr(d.d_state);
for (j = 0; j < d.d_nstates; j++, s.inc())
{
fixstate(g, s[0]);
}
}
private static int classify(parser_state ps, int type, CharPtr str)
{
grammar g = ps.p_grammar;
int n = g.g_ll.ll_nlabels;
if (type == NAME)
{
CharPtr s = new CharPtr(str);
labelPtr l = new labelPtr(g.g_ll.ll_label);
int i;
for (i = n; i > 0; i--, l.inc())
{
if (l[0].lb_type == NAME && l[0].lb_str != null &&
l[0].lb_str[0] == s[0] &&
strcmp(l[0].lb_str, s) == 0)
{
if (0 == ps.p_generators &&
s[0] == 'y' &&
strcmp(s, "yield") == 0)
{
break;
}
if (D())
{
printf("It's a keyword\n");
}
return(n - i);
}
}
}
{
labelPtr l = new labelPtr(g.g_ll.ll_label);
int i;
for (i = n; i > 0; i--, l.inc())
{
if (l[0].lb_type == type && l[0].lb_str == null)
{
if (D())
{
printf("It's a token we know\n");
}
return(n - i);
}
}
}
if (D())
{
printf("Illegal token\n");
}
return(-1);
}
//20170403
//#include "pgenheaders.h"
//#include "grammar.h"
//#include "node.h"
//#include "token.h"
//#include "parser.h"
// static void fixdfa(grammar *, dfa *);
// static void fixstate(grammar *, state *);
public static void PyGrammar_AddAccelerators(grammar g)
{
dfaPtr d;
int i;
#if _DEBUG
fprintf(stderr, "Adding parser accelerators ...\n");
#endif
d = new dfaPtr(g.g_dfa);
for (i = g.g_ndfas; --i >= 0; d.inc())
{
fixdfa(g, d[0]);
}
g.g_accel = 1;
#if _DEBUG
fprintf(stderr, "Done.\n");
#endif
}
public static void PyGrammar_RemoveAccelerators(grammar g)
{
dfaPtr d;
int i;
g.g_accel = 0;
d = new dfaPtr(g.g_dfa);
for (i = g.g_ndfas; --i >= 0; d.inc())
{
statePtr s;
int j;
s = new statePtr(d[0].d_state);
for (j = 0; j < d[0].d_nstates; j++, s.inc())
{
if (null != s[0].s_accel)
{
PyMem_DEL(ref s[0].s_accel);
}
s[0].s_accel = null;
}
}
}
private static ParsingGrammar Translate(grammar grammar)
{
var g = new ParsingGrammar();
if (grammar.header.nsName != null)
{
g.Name = string.Join(".", grammar.header.nsName.identifiers.Select(s => s.@string));
}
g.StartRuleName = grammar.header.identifiers.First().@string;
if (grammar.header.identifiers.Length > 1)
{
g.SkipRuleName = grammar.header.identifiers.Last().@string;
}
foreach (var r in grammar.rules)
{
g.Add(r.identifier.@string, TranslateSeq(r.exprsSeq));
}
return(g);
}
public static node PyParser_ParseFileFlags(FILEPtr fp, CharPtr filename, grammar g, int start,
CharPtr ps1, CharPtr ps2, perrdetail err_ret, int flags)
{
tok_state tok;
initerr(err_ret, filename);
if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == null)
{
err_ret.error = E_NOMEM;
return(null);
}
if (0 != Py_TabcheckFlag || 0 != Py_VerboseFlag)
{
tok.filename = filename;
tok.altwarning = (filename != null)?1:0;
if (Py_TabcheckFlag >= 2)
{
tok.alterror++;
}
}
return(parsetok(tok, g, start, err_ret, flags));
}
public static node PyParser_ParseStringFlags(CharPtr s, grammar g, int start,
perrdetail err_ret, int flags)
{
tok_state tok;
initerr(err_ret, null);
if ((tok = PyTokenizer_FromString(s)) == null)
{
err_ret.error = E_NOMEM;
return(null);
}
if (0 != Py_TabcheckFlag || 0 != Py_VerboseFlag)
{
tok.filename = "<string>";
tok.altwarning = (tok.filename != null)?1:0;
if (Py_TabcheckFlag >= 2)
{
tok.alterror++;
}
}
return(parsetok(tok, g, start, err_ret, flags));
}
private static void fixstate(grammar g, state s)
{
arcPtr a;
int k;
intPtr accel;
int nl = g.g_ll.ll_nlabels;
s.s_accept = 0;
accel = PyMem_NEW_int(nl);
for (k = 0; k < nl; k++)
{
accel[k] = -1;
}
a = new arcPtr(s.s_arc);
for (k = s.s_narcs; --k >= 0; a.inc())
{
int lbl = a[0].a_lbl;
label l = g.g_ll.ll_label[lbl];
int type = l.lb_type;
if (a[0].a_arrow >= (1 << 7))
{
printf("XXX too many states!\n");
continue;
}
if (ISNONTERMINAL(type))
{
dfa d1 = PyGrammar_FindDFA(g, type);
int ibit;
if (type - NT_OFFSET >= (1 << 7))
{
printf("XXX too high nonterminal number!\n");
continue;
}
for (ibit = 0; ibit < g.g_ll.ll_nlabels; ibit++)
{
if (testbit(d1.d_first, ibit))
{
if (accel[ibit] != -1)
{
printf("XXX ambiguity!\n");
}
accel[ibit] = a[0].a_arrow | (1 << 7) |
((type - NT_OFFSET) << 8);
}
}
}
else if (lbl == EMPTY)
{
s.s_accept = 1;
}
else if (lbl >= 0 && lbl < nl)
{
accel[lbl] = a[0].a_arrow;
}
}
while (nl > 0 && accel[nl - 1] == -1)
{
nl--;
}
for (k = 0; k < nl && accel[k] == -1;)
{
k++;
}
if (k < nl)
{
int i;
s.s_accel = PyMem_NEW_int(nl - k);
if (s.s_accel == null)
{
fprintf(stderr, "no mem to add parser accelerators\n");
exit(1);
}
s.s_lower = k;
s.s_upper = nl;
for (i = 0; k < nl; i++, k++)
{
s.s_accel[i] = accel[k];
}
}
PyMem_DEL(ref accel);
}
private static node parsetok(tok_state tok, grammar g, int start, perrdetail err_ret, int flags)
{
parser_state ps;
node n;
int started = 0;
if ((ps = PyParser_New(g, start)) == null)
{
fprintf(stderr, "no mem for new parser\n");
err_ret.error = E_NOMEM;
return(null);
}
if (0 != (flags & PyPARSE_YIELD_IS_KEYWORD))
{
ps.p_generators = 1;
}
for (;;)
{
CharPtr a = null, b = null;
int type;
size_t len;
CharPtr str;
type = PyTokenizer_Get(tok, ref a, ref b);
if (type == ERRORTOKEN)
{
err_ret.error = tok.done;
break;
}
if (type == ENDMARKER && 0 != started)
{
type = NEWLINE;
started = 0;
}
else
{
started = 1;
}
len = (size_t)(b - a);
str = PyMem_NEW_char2((int)(len + 1));
if (str == null)
{
fprintf(stderr, "no mem for next token\n");
err_ret.error = E_NOMEM;
break;
}
if (len > 0)
{
strncpy(str, a, (int)len);
}
str[len] = '\0';
if (type == NAME && 0 == ps.p_generators &&
len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0)
{
PySys_WriteStderr(yield_msg,
err_ret.filename == null ?
"<string>" : err_ret.filename,
tok.lineno);
}
if ((err_ret.error =
PyParser_AddToken(ps, (int)type, str, tok.lineno,
ref err_ret.expected)) != E_OK)
{
if (err_ret.error != E_DONE)
{
PyMem_DEL(str);
}
break;
}
}
if (err_ret.error == E_DONE)
{
n = ps.p_tree;
ps.p_tree = null;
}
else
{
n = null;
}
PyParser_Delete(ps);
if (n == null)
{
if (tok.lineno <= 1 && tok.done == E_EOF)
{
err_ret.error = E_EOF;
}
err_ret.lineno = tok.lineno;
err_ret.offset = tok.cur - tok.buf;
if (tok.buf != null)
{
size_t len = (size_t)(tok.inp - tok.buf);
err_ret.text = PyMem_NEW_char2((int)(len + 1));
if (err_ret.text != null)
{
if (len > 0)
{
strncpy(err_ret.text, tok.buf, (int)len);
}
err_ret.text[len] = '\0';
}
}
}
PyTokenizer_Free(tok);
return(n);
}
public static node PyParser_ParseFile(FILEPtr fp, CharPtr filename, grammar g, int start,
CharPtr ps1, CharPtr ps2, perrdetail err_ret)
{
return(PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
err_ret, 0));
}
//static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
//static void initerr(perrdetail *err_ret, char* filename);
public static node PyParser_ParseString(CharPtr s, grammar g, int start, perrdetail err_ret)
{
return(PyParser_ParseStringFlags(s, g, start, err_ret, 0));
}