public TokenEnumerator(FA lexer, IEnumerable <char> @string, bool reportEndToken)
{
_lexer = lexer;
_input = @string.GetEnumerator();
_buffer = new StringBuilder();
_reportEndToken = reportEndToken;
_initialStates = _lexer.FillEpsilonClosure();
Reset(); // Reset is used here to initialize the rest of the values
}
static FA _Determinize(FA fa, IProgress <FAProgress> progress = null)
{
if (null != progress)
{
progress.Report(new FAProgress(FAStatus.DfaTransform, 0));
}
var p = new HashSet <int>();
var closure = new List <FA>();
fa.FillClosure(closure);
for (int ic = closure.Count, i = 0; i < ic; ++i)
{
var ffa = closure[i];
p.Add(0);
foreach (var t in ffa.InputTransitions)
{
p.Add(t.Key.Key);
if (t.Key.Value < 0x10ffff)
{
p.Add((t.Key.Value + 1));
}
}
}
var points = new int[p.Count];
p.CopyTo(points, 0);
Array.Sort(points);
var comparer = _SetComparer.Default;
var sets = new Dictionary <HashSet <FA>, HashSet <FA> >(comparer);
var working = new Queue <HashSet <FA> >();
var dfaMap = new Dictionary <HashSet <FA>, FA>(comparer);
var initial = new HashSet <FA>();
fa.FillEpsilonClosure(initial);
sets.Add(initial, initial);
working.Enqueue(initial);
var result = new FA();
foreach (var afa in initial)
{
if (afa.IsAccepting)
{
result.IsAccepting = true;
result.AcceptSymbol = afa.AcceptSymbol;
break;
}
}
dfaMap.Add(initial, result);
var j = 1;
while (working.Count > 0)
{
var s = working.Dequeue();
var ecs = FillEpsilonClosure(s);
FA dfa;
dfaMap.TryGetValue(s, out dfa);
foreach (FA q in ecs)
{
if (q.IsAccepting)
{
dfa.IsAccepting = true;
dfa.AcceptSymbol = q.AcceptSymbol;
break;
}
}
for (var i = 0; i < points.Length; i++)
{
var set = new HashSet <FA>();
foreach (FA c in ecs)
{
foreach (var trns in c.InputTransitions)
{
if (trns.Key.Key <= points[i] && points[i] <= trns.Key.Value)
{
foreach (var efa in trns.Value.FillEpsilonClosure())
{
set.Add(efa);
}
}
}
}
if (!sets.ContainsKey(set))
{
sets.Add(set, set);
working.Enqueue(set);
dfaMap.Add(set, new FA());
}
FA dst;
dfaMap.TryGetValue(set, out dst);
int first = points[i];
int last;
if (i + 1 < points.Length)
{
last = (points[i + 1] - 1);
}
else
{
last = 0x10ffff;
}
dfa.InputTransitions.Add(new KeyValuePair <int, int>(first, last), dst);
}
if (null != progress)
{
progress.Report(new FAProgress(FAStatus.DfaTransform, j));
}
++j;
}
// remove dead transitions
foreach (var ffa in result.FillClosure())
{
var itrns = new List <KeyValuePair <KeyValuePair <int, int>, FA> >(ffa.InputTransitions);
foreach (var trns in itrns)
{
if (null == trns.Value.FirstAcceptingState)
{
ffa.InputTransitions.Remove(trns.Key);
}
}
if (null != progress)
{
progress.Report(new FAProgress(FAStatus.DfaTransform, j));
}
++j;
}
return(result);
}