internal void AddMove(NfaState newState)
{
if (!epsilonMoves.Contains(newState))
InsertInOrder(epsilonMoves, newState);
}
private static void DumpCompositeStatesAsciiMoves(TextWriter ostr, string key, int byteNum, bool[] dumped)
{
int i;
int[] nameSet = allNextStates[key];
if (nameSet.Length == 1 || dumped[StateNameForComposite(key)])
return;
NfaState toBePrinted = null;
int neededStates = 0;
NfaState tmp;
NfaState stateForCase = null;
String toPrint = "";
bool stateBlock = stateBlockTable.ContainsKey(key);
for (i = 0; i < nameSet.Length; i++) {
tmp = allStates[nameSet[i]];
if (tmp.asciiMoves[byteNum] != 0L) {
if (neededStates++ == 1)
break;
else
toBePrinted = tmp;
} else
dumped[tmp.stateName] = true;
if (tmp.stateForCase != null) {
if (stateForCase != null)
throw new InvalidOperationException("CSharpCC Bug: Please send a report: ");
stateForCase = tmp.stateForCase;
}
}
if (stateForCase != null)
toPrint = stateForCase.PrintNoBreak(ostr, byteNum, dumped);
if (neededStates == 0) {
if (stateForCase != null && toPrint.Equals(""))
ostr.WriteLine(" break;");
return;
}
if (neededStates == 1) {
if (!toPrint.Equals(""))
ostr.Write(toPrint);
ostr.WriteLine(" case " + StateNameForComposite(key) + ":");
if (!dumped[toBePrinted.stateName] && !stateBlock && toBePrinted.inNextOf > 1)
ostr.WriteLine(" case " + toBePrinted.stateName + ":");
dumped[toBePrinted.stateName] = true;
toBePrinted.DumpAsciiMove(ostr, byteNum, dumped);
return;
}
var partition = PartitionStatesSetForAscii(nameSet, byteNum);
if (!toPrint.Equals(""))
ostr.Write(toPrint);
int keyState = StateNameForComposite(key);
ostr.WriteLine(" case " + keyState + ":");
if (keyState < generatedStates)
dumped[keyState] = true;
for (i = 0; i < partition.Count; i++) {
var subSet = partition[i];
for (int j = 0; j < subSet.Count; j++) {
tmp = subSet[j];
if (stateBlock)
dumped[tmp.stateName] = true;
tmp.DumpAsciiMoveForCompositeState(ostr, byteNum, j != 0);
}
}
if (stateBlock)
ostr.WriteLine(" break;");
else
ostr.WriteLine(" break;");
}
private void MergeMoves(NfaState other)
{
// Warning : This function does not merge epsilon moves
if (asciiMoves == other.asciiMoves) {
CSharpCCErrors.SemanticError("Bug in CSharpCC : Please send " +
"a report along with the input that caused this. Thank you.");
throw new InvalidOperationException();
}
asciiMoves[0] = asciiMoves[0] | other.asciiMoves[0];
asciiMoves[1] = asciiMoves[1] | other.asciiMoves[1];
if (other.charMoves != null) {
if (charMoves == null)
charMoves = other.charMoves;
else {
char[] tmpCharMoves = new char[charMoves.Length +
other.charMoves.Length];
Array.Copy(charMoves, 0, tmpCharMoves, 0, charMoves.Length);
charMoves = tmpCharMoves;
for (int i = 0; i < other.charMoves.Length; i++)
AddChar(other.charMoves[i]);
}
}
if (other.rangeMoves != null) {
if (rangeMoves == null)
rangeMoves = other.rangeMoves;
else {
char[] tmpRangeMoves = new char[rangeMoves.Length +
other.rangeMoves.Length];
Array.Copy(rangeMoves, 0, tmpRangeMoves, 0, rangeMoves.Length);
rangeMoves = tmpRangeMoves;
for (int i = 0; i < other.rangeMoves.Length; i += 2)
AddRange(other.rangeMoves[i], other.rangeMoves[i + 1]);
}
}
if (other.kind < kind)
kind = other.kind;
if (other.kindToPrint < kindToPrint)
kindToPrint = other.kindToPrint;
isFinal |= other.isFinal;
}
public static int moveFromSetForRegEx(char c, NfaState[] states, NfaState[] newStates, int round)
{
int start = 0;
int sz = states.Length;
for (int i = 0; i < sz; i++) {
NfaState tmp1, tmp2;
if ((tmp1 = states[i]) == null)
break;
if (tmp1.CanMoveUsingChar(c)) {
if (tmp1.kindToPrint != Int32.MaxValue) {
newStates[start] = null;
return 1;
}
NfaState[] v = tmp1.next.epsilonMoveArray;
for (int j = v.Length; j-- > 0;) {
if ((tmp2 = v[j]).round != round) {
tmp2.round = round;
newStates[start++] = tmp2;
}
}
}
}
newStates[start] = null;
return Int32.MaxValue;
}
private static void InsertInOrder(IList<NfaState> v, NfaState s)
{
int j;
for (j = 0; j < v.Count; j++)
if (v[j].id > s.id)
break;
else if (v[j].id == s.id)
return;
v.Insert(j, s);
}
private NfaState CreateClone()
{
NfaState retVal = new NfaState();
retVal.isFinal = isFinal;
retVal.kind = kind;
retVal.lookingFor = lookingFor;
retVal.lexState = lexState;
retVal.inNextOf = inNextOf;
retVal.MergeMoves(this);
return retVal;
}
public Nfa() {
Start = new NfaState();
End = new NfaState();
}
private static void FindStatesWithNoBreak()
{
Dictionary<string, string> printed = new Dictionary<string, string>();
bool[] put = new bool[generatedStates];
int cnt = 0;
int i, j, foundAt = 0;
for (j = 0; j < allStates.Count; j++) {
NfaState stateForCase = null;
NfaState tmpState = allStates[j];
if (tmpState.stateName == -1 || tmpState.dummy || !tmpState.UsefulState() ||
tmpState.next == null || tmpState.next.usefulEpsilonMoves < 1)
continue;
String s = tmpState.next.epsilonMovesString;
if (compositeStateTable.ContainsKey(s) ||
printed.ContainsKey(s))
continue;
printed[s] = s;
int[] nexts = allNextStates[s];
if (nexts.Length == 1)
continue;
int state = cnt;
//System.out.println("State " + tmpState.stateName + " : " + s);
for (i = 0; i < nexts.Length; i++) {
if ((state = nexts[i]) == -1)
continue;
NfaState tmp = allStates[state];
if (!tmp.isComposite && tmp.inNextOf == 1) {
if (put[state])
throw new InvalidOperationException("CSharpCC Bug: Please send a report");
foundAt = i;
cnt++;
stateForCase = tmp;
put[state] = true;
//System.out.print(state + " : " + tmp.inNextOf + ", ");
break;
}
}
//System.out.println("");
if (stateForCase == null)
continue;
for (i = 0; i < nexts.Length; i++) {
if ((state = nexts[i]) == -1)
continue;
NfaState tmp = allStates[state];
if (!put[state] && tmp.inNextOf > 1 && !tmp.isComposite && tmp.stateForCase == null) {
cnt++;
nexts[i] = -1;
put[state] = true;
int toSwap = nexts[0];
nexts[0] = nexts[foundAt];
nexts[foundAt] = toSwap;
tmp.stateForCase = stateForCase;
stateForCase.stateForCase = tmp;
stateSetsToFix[s] = nexts;
//System.out.println("For : " + s + "; " + stateForCase.stateName +
//" and " + tmp.stateName);
goto Outer;
}
}
for (i = 0; i < nexts.Length; i++) {
if ((state = nexts[i]) == -1)
continue;
NfaState tmp = (NfaState) allStates[state];
if (tmp.inNextOf <= 1)
put[state] = false;
}
Outer:
;
}
}
internal Nfa(NfaState start, NfaState end) {
Start = start;
End = end;
}
internal static void GenerateNfaStartStates(TextWriter ostr, NfaState initialState)
{
bool[] seen = new bool[NfaState.generatedStates];
IDictionary<string, string> stateSets = new Dictionary<string, string>();
String stateSetString = "";
int i, j, kind, jjmatchedPos = 0;
int maxKindsReqd = maxStrKind/64 + 1;
long[] actives;
IList<NfaState> newStates = new List<NfaState>();
IList<NfaState> oldStates = null, jjtmpStates;
statesForPos = new IDictionary<string, long[]>[maxLen];
intermediateKinds = new int[maxStrKind + 1][];
intermediateMatchedPos = new int[maxStrKind + 1][];
for (i = 0; i < maxStrKind; i++) {
if (LexGen.lexStates[i] != LexGen.lexStateIndex)
continue;
String image = allImages[i];
if (image == null || image.Length < 1)
continue;
try {
if ((oldStates = new List<NfaState>(initialState.epsilonMoves)).Count == 0) {
DumpNfaStartStatesCode(statesForPos, ostr);
return;
}
} catch (Exception e) {
CSharpCCErrors.SemanticError("Error cloning state vector");
}
intermediateKinds[i] = new int[image.Length];
intermediateMatchedPos[i] = new int[image.Length];
jjmatchedPos = 0;
kind = Int32.MaxValue;
for (j = 0; j < image.Length; j++) {
if (oldStates == null || oldStates.Count <= 0) {
// Here, j > 0
kind = intermediateKinds[i][j] = intermediateKinds[i][j - 1];
jjmatchedPos = intermediateMatchedPos[i][j] = intermediateMatchedPos[i][j - 1];
} else {
kind = NfaState.MoveFromSet(image[j], oldStates, newStates);
oldStates.Clear();
if (j == 0 && kind != Int32.MaxValue &&
LexGen.canMatchAnyChar[LexGen.lexStateIndex] != -1 &&
kind > LexGen.canMatchAnyChar[LexGen.lexStateIndex])
kind = LexGen.canMatchAnyChar[LexGen.lexStateIndex];
if (GetStrKind(image.Substring(0, j + 1)) < kind) {
intermediateKinds[i][j] = kind = Int32.MaxValue;
jjmatchedPos = 0;
} else if (kind != Int32.MaxValue) {
intermediateKinds[i][j] = kind;
jjmatchedPos = intermediateMatchedPos[i][j] = j;
} else if (j == 0)
kind = intermediateKinds[i][j] = Int32.MaxValue;
else {
kind = intermediateKinds[i][j] = intermediateKinds[i][j - 1];
jjmatchedPos = intermediateMatchedPos[i][j] = intermediateMatchedPos[i][j - 1];
}
stateSetString = NfaState.GetStateSetString(newStates);
}
if (kind == Int32.MaxValue &&
(newStates == null || newStates.Count == 0))
continue;
int p;
if (stateSets.ContainsKey(stateSetString)) {
stateSets[stateSetString] = stateSetString;
for (p = 0; p < newStates.Count; p++) {
if (seen[newStates[p].stateName])
newStates[p].inNextOf++;
else
seen[newStates[p].stateName] = true;
}
} else {
for (p = 0; p < newStates.Count; p++)
seen[newStates[p].stateName] = true;
}
jjtmpStates = oldStates;
oldStates = newStates;
(newStates = jjtmpStates).Clear();
if (statesForPos[j] == null)
statesForPos[j] = new Dictionary<string, long[]>();
if (!(statesForPos[j].TryGetValue(kind + ", " + jjmatchedPos + ", " + stateSetString, out actives))) {
actives = new long[maxKindsReqd];
statesForPos[j][kind + ", " + jjmatchedPos + ", " + stateSetString] = actives;
}
actives[i/64] |= 1L << (i%64);
//String name = NfaState.StoreStateSet(stateSetString);
}
}
DumpNfaStartStatesCode(statesForPos, ostr);
}
public override Nfa GenerateNfa(bool ignoreCase)
{
if (Image.Length == 1) {
RCharacterList temp = new RCharacterList(Image[0]);
return temp.GenerateNfa(ignoreCase);
}
NfaState startState = new NfaState();
NfaState theStartState = startState;
NfaState finalState = null;
if (Image.Length == 0)
return new Nfa(theStartState, theStartState);
int i;
for (i = 0; i < Image.Length; i++) {
finalState = new NfaState();
startState.charMoves = new char[1];
startState.AddChar(Image[i]);
if (Options.getIgnoreCase() || ignoreCase) {
startState.AddChar(Char.ToLower(Image[i]));
startState.AddChar(Char.ToUpper(Image[i]));
}
startState.next = finalState;
startState = finalState;
}
return new Nfa(theStartState, finalState);
}
public static void start()
{
if (!Options.getBuildTokenManager() ||
Options.getUserTokenManager() ||
CSharpCCErrors.ErrorCount > 0)
return;
keepLineCol = Options.getKeepLineColumn();
List<RegularExpression> choices = new List<RegularExpression>();
IEnumerator e;
TokenProduction tp;
int i, j;
staticString = (Options.getStatic() ? "static " : "");
tokMgrClassName = CSharpCCGlobals.cu_name + "TokenManager";
PrintClassHead();
BuildLexStatesTable();
e = allTpsForState.Keys.GetEnumerator();
bool ignoring = false;
while (e.MoveNext()) {
NfaState.ReInit();
RStringLiteral.ReInit();
String key = (String) e.Current;
lexStateIndex = GetIndex(key);
lexStateSuffix = "_" + lexStateIndex;
IList<TokenProduction> allTps = allTpsForState[key];
initStates[key] = initialState = new NfaState();
ignoring = false;
singlesToSkip[lexStateIndex] = new NfaState();
singlesToSkip[lexStateIndex].dummy = true;
if (key.Equals("DEFAULT"))
defaultLexState = lexStateIndex;
for (i = 0; i < allTps.Count; i++) {
tp = allTps[i];
int kind = tp.Kind;
bool ignore = tp.IgnoreCase;
IList<RegExprSpec> rexps = tp.RegexSpecs;
if (i == 0)
ignoring = ignore;
for (j = 0; j < rexps.Count; j++) {
RegExprSpec respec = rexps[j];
curRE = respec.RegularExpression;
rexprs[curKind = curRE.Ordinal] = curRE;
lexStates[curRE.Ordinal] = lexStateIndex;
ignoreCase[curRE.Ordinal] = ignore;
if (curRE.IsPrivate) {
kinds[curRE.Ordinal] = -1;
continue;
}
if (curRE is RStringLiteral &&
!((RStringLiteral) curRE).Image.Equals("")) {
((RStringLiteral) curRE).GenerateDfa(ostr, curRE.Ordinal);
if (i != 0 && !mixed[lexStateIndex] && ignoring != ignore)
mixed[lexStateIndex] = true;
} else if (curRE.CanMatchAnyChar) {
if (canMatchAnyChar[lexStateIndex] == -1 ||
canMatchAnyChar[lexStateIndex] > curRE.Ordinal)
canMatchAnyChar[lexStateIndex] = curRE.Ordinal;
} else {
Nfa temp;
if (curRE is RChoice)
choices.Add(curRE);
temp = curRE.GenerateNfa(ignore);
temp.End.isFinal = true;
temp.End.kind = curRE.Ordinal;
initialState.AddMove(temp.Start);
}
if (kinds.Length < curRE.Ordinal) {
int[] tmp = new int[curRE.Ordinal + 1];
Array.Copy(kinds, 0, tmp, 0, kinds.Length);
kinds = tmp;
}
//System.out.println(" ordina : " + curRE.ordinal);
kinds[curRE.Ordinal] = kind;
if (respec.NextState != null &&
!respec.NextState.Equals(lexStateName[lexStateIndex]))
newLexState[curRE.Ordinal] = respec.NextState;
if (respec.Action != null && respec.Action.ActionTokens != null &&
respec.Action.ActionTokens.Count > 0)
actions[curRE.Ordinal] = respec.Action;
switch (kind) {
case TokenProduction.SPECIAL:
hasSkipActions |= (actions[curRE.Ordinal] != null) ||
(newLexState[curRE.Ordinal] != null);
hasSpecial = true;
toSpecial[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64);
toSkip[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64);
break;
case TokenProduction.SKIP:
hasSkipActions |= (actions[curRE.Ordinal] != null);
hasSkip = true;
toSkip[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64);
break;
case TokenProduction.MORE:
hasMoreActions |= (actions[curRE.Ordinal] != null);
hasMore = true;
toMore[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64);
if (newLexState[curRE.Ordinal] != null)
canReachOnMore[GetIndex(newLexState[curRE.Ordinal])] = true;
else
canReachOnMore[lexStateIndex] = true;
break;
case TokenProduction.TOKEN:
hasTokenActions |= (actions[curRE.Ordinal] != null);
toToken[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64);
break;
}
}
}
// Generate a static block for initializing the nfa transitions
NfaState.ComputeClosures();
for (i = 0; i < initialState.epsilonMoves.Count; i++)
initialState.epsilonMoves[i].GenerateCode();
if (hasNfa[lexStateIndex] = (NfaState.generatedStates != 0)) {
initialState.GenerateCode();
initialState.GenerateInitMoves(ostr);
}
if (initialState.kind != Int32.MaxValue && initialState.kind != 0) {
if ((toSkip[initialState.kind/64] & (1L << initialState.kind)) != 0L ||
(toSpecial[initialState.kind/64] & (1L << initialState.kind)) != 0L)
hasSkipActions = true;
else if ((toMore[initialState.kind/64] & (1L << initialState.kind)) != 0L)
hasMoreActions = true;
else
hasTokenActions = true;
if (initMatch[lexStateIndex] == 0 ||
initMatch[lexStateIndex] > initialState.kind) {
initMatch[lexStateIndex] = initialState.kind;
hasEmptyMatch = true;
}
} else if (initMatch[lexStateIndex] == 0)
initMatch[lexStateIndex] = Int32.MaxValue;
RStringLiteral.FillSubString();
if (hasNfa[lexStateIndex] && !mixed[lexStateIndex])
RStringLiteral.GenerateNfaStartStates(ostr, initialState);
RStringLiteral.DumpDfaCode(ostr);
if (hasNfa[lexStateIndex])
NfaState.DumpMoveNfa(ostr);
if (stateSetSize < NfaState.generatedStates)
stateSetSize = NfaState.generatedStates;
}
for (i = 0; i < choices.Count; i++)
((RChoice) choices[i]).CheckUnmatchability();
NfaState.DumpStateSets(ostr);
CheckEmptyStringMatch();
NfaState.DumpNonAsciiMoveMethods(ostr);
RStringLiteral.DumpStrLiteralImages(ostr);
DumpStaticVarDeclarations();
DumpFillToken();
DumpGetNextToken();
if (Options.getDebugTokenManager()) {
NfaState.DumpStatesForKind(ostr);
DumpDebugMethods();
}
if (hasLoop) {
ostr.WriteLine("{0}int[] ccEmptyLineNo = new int[{1}];", staticString, maxLexStates);
ostr.WriteLine("{0}int[] ccEmptyColNo = new int[{1}];", staticString, maxLexStates);
ostr.WriteLine("{0}bool[] ccBeenHere = new bool[{1}];", staticString, maxLexStates);
}
if (hasSkipActions)
DumpSkipActions();
if (hasMoreActions)
DumpMoreActions();
if (hasTokenActions)
DumpTokenActions();
NfaState.PrintBoilerPlate(ostr);
ostr.WriteLine( /*{*/ "}");
if (namespaceInserted)
ostr.WriteLine("}");
ostr.Close();
}
public static void reInit()
{
ostr = null;
staticString = null;
tokMgrClassName = null;
allTpsForState = new Dictionary<string, IList<TokenProduction>>();
lexStateIndex = 0;
kinds = null;
maxOrdinal = 1;
lexStateSuffix = null;
newLexState = null;
lexStates = null;
ignoreCase = null;
actions = null;
initStates = new Dictionary<string, NfaState>();
stateSetSize = 0;
maxLexStates = 0;
lexStateName = null;
singlesToSkip = null;
toSkip = null;
toSpecial = null;
toMore = null;
toToken = null;
defaultLexState = 0;
rexprs = null;
maxLongsReqd = null;
initMatch = null;
canMatchAnyChar = null;
hasEmptyMatch = false;
canLoop = null;
stateHasActions = null;
hasLoop = false;
canReachOnMore = null;
hasNfa = null;
mixed = null;
initialState = null;
curKind = 0;
hasSkipActions = false;
hasMoreActions = false;
hasTokenActions = false;
hasSpecial = false;
hasSkip = false;
hasMore = false;
curRE = null;
}
