static void _EnqueueFiber(ref int lcount, ref TokenizerFiber[] l, TokenizerFiber t, int sp) { if ((l.Length <= lcount)) { TokenizerFiber[] newarr = new TokenizerFiber[(l.Length * 2)]; System.Array.Copy(l, 0, newarr, 0, l.Length); l = newarr; } l[lcount] = t; lcount = (lcount + 1); int[] pc = t.Program[t.Index]; int op = pc[0]; if ((TokenizerEnumerator._Jmp == op)) { TokenizerEnumerator._EnqueueFiber(ref lcount, ref l, new TokenizerFiber(t, pc[1], t.Saved), sp); return; } if ((TokenizerEnumerator._Split == op)) { for (int j = 1; (j < pc.Length); j = (j + 1)) { TokenizerEnumerator._EnqueueFiber(ref lcount, ref l, new TokenizerFiber(t.Program, pc[j], t.Saved), sp); } return; } if ((TokenizerEnumerator._Save == op)) { int slot = pc[1]; int max = t.Saved.Length; if ((slot > max)) { max = slot; } int[] saved = new int[max]; for (int i = 0; (i < t.Saved.Length); i = (i + 1)) { saved[i] = t.Saved[i]; } saved[slot] = sp; TokenizerEnumerator._EnqueueFiber(ref lcount, ref l, new TokenizerFiber(t, (t.Index + 1), saved), sp); return; } }
public TokenizerFiber(TokenizerFiber fiber, int index, int[] saved) { this.Program = fiber.Program; this.Index = index; this.Saved = saved; }
int _Lex() { this._capture.Clear(); if ((this._state == TokenizerEnumerator._BeforeBegin)) { this._MoveNextInput(); } int i; int match = -1; TokenizerFiber[] tmp; int currentFiberCount = 0; int nextFiberCount = 0; int[] pc; int sp = 0; int[] saved; int[] matched; saved = new int[2]; TokenizerEnumerator._EnqueueFiber(ref currentFiberCount, ref this._currentFibers, new TokenizerFiber(this._program, 0, saved), 0); matched = null; int cur = -1; if ((false == (TokenizerEnumerator._EndOfInput == this._ch))) { char ch1 = ((char)(this._ch)); if (char.IsHighSurrogate(ch1)) { if ((false == this._MoveNextInput())) { throw new IOException(string.Format("Expecting low surrogate in unicode stream. The input source is corrupt or not val" + "id Unicode at line {0}, column {1}, position {2}", this._line, this._column, this._position)); } this._column = (this._column - 1); char ch2 = ((char)(this._ch)); cur = char.ConvertToUtf32(ch1, ch2); } else { cur = ch1; } } else { cur = -1; } for ( ; (0 < currentFiberCount); ) { bool passed = false; for (i = 0; (i < currentFiberCount); i = (i + 1)) { TokenizerFiber t = this._currentFibers[i]; pc = t.Program[t.Index]; saved = t.Saved; int op = pc[0]; if ((TokenizerEnumerator._Switch == op)) { int idx = 1; for ( ; ((idx < pc.Length) && (-2 < pc[idx])); ) { if (TokenizerEnumerator._InRanges(pc, ref idx, cur)) { for ( ; (false == (-1 == pc[idx])); ) { idx = (idx + 1); } idx = (idx + 1); passed = true; TokenizerEnumerator._EnqueueFiber(ref nextFiberCount, ref this._nextFibers, new TokenizerFiber(t, pc[idx], saved), (sp + 1)); idx = pc.Length; } else { for ( ; (false == (-1 == pc[idx])); ) { idx = (idx + 1); } idx = (idx + 1); } idx = (idx + 1); } if (((idx < pc.Length) && (-2 == pc[idx]))) { idx = (idx + 1); for ( ; (idx < pc.Length); ) { TokenizerEnumerator._EnqueueFiber(ref nextFiberCount, ref this._nextFibers, new TokenizerFiber(t, pc[idx], saved), sp); idx = (idx + 1); } } } if ((TokenizerEnumerator._Char == op)) { if ((cur == pc[1])) { passed = true; TokenizerEnumerator._EnqueueFiber(ref nextFiberCount, ref this._nextFibers, new TokenizerFiber(t, (t.Index + 1), saved), (sp + 1)); } } else { if ((TokenizerEnumerator._Set == op)) { if (TokenizerEnumerator._InRanges(pc, cur)) { passed = true; TokenizerEnumerator._EnqueueFiber(ref nextFiberCount, ref this._nextFibers, new TokenizerFiber(t, (t.Index + 1), saved), (sp + 1)); } } else { if ((TokenizerEnumerator._NSet == op)) { if (((false == TokenizerEnumerator._InRanges(pc, cur)) && (false == (TokenizerEnumerator._EndOfInput == this._ch)))) { passed = true; TokenizerEnumerator._EnqueueFiber(ref nextFiberCount, ref this._nextFibers, new TokenizerFiber(t, (t.Index + 1), saved), (sp + 1)); } } else { if ((TokenizerEnumerator._UCode == op)) { string str = char.ConvertFromUtf32(cur); if ((((int)(char.GetUnicodeCategory(str, 0))) == pc[1])) { passed = true; TokenizerEnumerator._EnqueueFiber(ref nextFiberCount, ref this._nextFibers, new TokenizerFiber(t, (t.Index + 1), saved), (sp + 1)); } } else { if ((TokenizerEnumerator._NUCode == op)) { string str = char.ConvertFromUtf32(cur); if (((false == (((int)(char.GetUnicodeCategory(str, 0))) == pc[1])) && (false == (TokenizerEnumerator._EndOfInput == this._ch)))) { passed = true; TokenizerEnumerator._EnqueueFiber(ref nextFiberCount, ref this._nextFibers, new TokenizerFiber(t, (t.Index + 1), saved), (sp + 1)); } } else { if ((TokenizerEnumerator._Any == op)) { if ((false == (TokenizerEnumerator._EndOfInput == this._ch))) { passed = true; TokenizerEnumerator._EnqueueFiber(ref nextFiberCount, ref this._nextFibers, new TokenizerFiber(t, (t.Index + 1), saved), (sp + 1)); } } else { if ((TokenizerEnumerator._Match == op)) { matched = saved; match = pc[1]; i = currentFiberCount; } } } } } } } } if (passed) { this._capture.Append(char.ConvertFromUtf32(cur)); this._MoveNextInput(); if ((false == (TokenizerEnumerator._EndOfInput == this._ch))) { char ch1 = ((char)(this._ch)); if (char.IsHighSurrogate(ch1)) { if ((false == this._MoveNextInput())) { throw new IOException(string.Format("Expecting low surrogate in unicode stream. The input source is corrupt or not val" + "id Unicode at line {0}, column {1}, position {2}", this._line, this._column, this._position)); } this._column = (this._column - 1); sp = (sp + 1); char ch2 = ((char)(this._ch)); cur = char.ConvertToUtf32(ch1, ch2); } else { cur = ch1; } } else { cur = -1; } sp = (sp + 1); } tmp = this._currentFibers; this._currentFibers = this._nextFibers; this._nextFibers = tmp; currentFiberCount = nextFiberCount; nextFiberCount = 0; } if ((null != matched)) { int start = matched[0]; int len = matched[1]; this._value = this._capture.ToString(start, (len - start)); return(match); } return(-1); }