internal CharacterRangeToken(bool inverted, char[] singleTons, Tuple <char, char>[] rangeSet, UnicodeCategory[] letterCategories, int length, int line, int column, long position)
: base(line, column, position)
{
this.ranges = RegularLanguageSet.GetRegularLanguageSet(inverted, singleTons, rangeSet, letterCategories);
this.length = length;
this.inverted = inverted;
}
private RegularLanguageNFAState Subtract(RegularLanguageNFAState left, RegularLanguageNFAState right)
{
/* *
* Include the right-element where the left overlaps it;
* however, every other point of the right-automation
* is ignored.
* *
* Where the left and right automation are at an edge
* point, the automation does not yield an edge due to
* the exclusive nature of the operation.
* */
RegularLanguageNFAState result = new RegularLanguageNFAState();
if (left.IsEdge)
{
if (right.IsEdge)
result.ForcedNoEdge = true;
else
result.IsEdge = true;
}
foreach (var transition in left.OutTransitions.Keys)
{
IDictionary<RegularLanguageSet, IFiniteAutomataTransitionNode<RegularLanguageSet, List<RegularLanguageNFAState>>> colliders;
RegularLanguageSet remainder = right.OutTransitions.GetColliders(transition, out colliders);
if (!remainder.IsEmpty)
foreach (var target in left.OutTransitions[transition])
result.MoveTo(remainder, target);
foreach (var collision in colliders.Keys)
foreach (var leftTarget in left.OutTransitions[transition])
foreach (var rightTarget in colliders[collision].Target)
result.MoveTo(collision, Subtract(leftTarget, rightTarget));
}
return result;
}
public IUnicodeTargetPartialCategory Add(UnicodeCategory category, RegularLanguageSet negativeAssertion)
{
UnicodeTargetPartialCategory result = new UnicodeTargetPartialCategory(category, negativeAssertion);
base._Add(category, result);
return(result);
}
public void BuildState(Dictionary <ITokenSource, Captures.ICaptureTokenStructuralItem> sourceReplacementLookup)
{
var thisReplacement = sourceReplacementLookup.ContainsKey(this) ? (ITokenSource)(sourceReplacementLookup[this]) : (ITokenSource)this;
var target = (InlinedTokenExpressionSeries)this.SearchTarget;
target.BuildState(sourceReplacementLookup);
RegularLanguageNFAState current = null;
Stack <RegularLanguageNFAState> states = new Stack <RegularLanguageNFAState>(new RegularLanguageNFAState[] { target.State });
List <RegularLanguageNFAState> covered = new List <RegularLanguageNFAState>();
states.Peek().SetInitial(thisReplacement);
//Step through the sequence until it's finished with the all states
//associated to the scan operation.
while (states.Count > 0)
{
current = states.Pop();
if (covered.Contains(current))
{
continue;
}
covered.Add(current);
RegularLanguageSet currentSet = current.OutTransitions.FullCheck;
if (!currentSet.IsEmpty)
{
foreach (var transition in current.OutTransitions.Values)
{
foreach (var transitionTarget in transition)
{
if (!covered.Contains(transitionTarget))
{
states.Push(transitionTarget);
}
}
}
currentSet = currentSet.Complement();
if (!(currentSet.IsEmpty))
{
current.MoveTo(currentSet, target.State);
}
}
}
state = target.State;
List <RegularLanguageNFAState> flatline = new List <RegularLanguageNFAState>();
RegularLanguageNFAState.FlatlineState(state, flatline);
foreach (var fState in flatline)
{
fState.SetIntermediate(thisReplacement);
}
state.SetInitial(thisReplacement);
foreach (var edge in State.ObtainEdges())
{
edge.SetFinal(thisReplacement);
}
state.HandleRepeatCycle <RegularLanguageSet, RegularLanguageNFAState, RegularLanguageDFAState, ITokenSource, RegularLanguageNFARootState, IInlinedTokenItem>(this, thisReplacement, OilexerGrammarInliningCore.TokenRootStateClonerCache, OilexerGrammarInliningCore.TokenStateClonerCache);
}
internal static void PropagateUnicodeCategories(ref RegularLanguageSet toStringArray, bool minimizeSet, out UnicodeCategory[] toStringUnicodeCategories)
{
var resultSet = new List <UnicodeCategory>();
foreach (var unicodeCategory in unicodeCategoryData.Keys)
{
var currentUnicodeSubset = unicodeCategoryData[unicodeCategory].Intersect(toStringArray);
if (currentUnicodeSubset.Equals(unicodeCategoryData[unicodeCategory]))
{
resultSet.Add(unicodeCategory);
toStringArray = toStringArray.SymmetricDifference(currentUnicodeSubset);
}
}
toStringArray.Reduce();
toStringUnicodeCategories = resultSet.ToArray();
}
private static void Breakdown(RegularLanguageNFAState targetState, Dictionary <char, List <string> > currentSubsets, RegularLanguageNFAState[] endTargets, bool caseSensitive)
{
foreach (var c in currentSubsets.Keys)
{
var currentSet = currentSubsets[c];
var transition = new RegularLanguageSet(caseSensitive, c);
List <string> characters = null;
Dictionary <char, List <string> > nextSubsets = new Dictionary <char, List <string> >();
foreach (var str in currentSet)
{
if (str == string.Empty)
{
foreach (var state in endTargets)
{
targetState.MoveTo(transition, state);
}
}
else
{
char first = str[0];
if (!nextSubsets.ContainsKey(first))
{
nextSubsets.Add(first, characters = new List <string>());
}
string substr = str.Substring(1);
if (!characters.Contains(substr))
{
characters.Add(substr);
}
}
}
if (nextSubsets.Count > 0)
{
var nextSubstate = new RegularLanguageNFAState();
targetState.MoveTo(transition, nextSubstate);
Breakdown(nextSubstate, nextSubsets, endTargets, caseSensitive);
}
}
}
public static void PropagateUnicodeCategoriesAndPartials(ref RegularLanguageSet target, RegularLanguageSet negativeNoise, out UnicodeCategory[] categories, out Dictionary <UnicodeCategory, RegularLanguageSet> categoriesWithNegativeAssertions)
{
List <UnicodeCategory> fullCategories = new List <UnicodeCategory>();
categoriesWithNegativeAssertions = new Dictionary <UnicodeCategory, RegularLanguageSet>();
foreach (var category in unicodeCategoryData.Keys)
{
var categoryData = unicodeCategoryData[category];
var intersection = categoryData & target;
int compared = (int)(intersection.CountTrue());//unicodeCategoryData[category].Compare(target);
if (compared == unicodeCategoryTrueCounts[category])
{
fullCategories.Add(category);
target = target.SymmetricDifference(intersection);
}
else if (compared > unicodeCategoryTrueCounts[category] - compared)
{
//if there are more elements that overlap than there are not,
//include this category with a negative assertion.
var categoryCopy = categoryData ^ intersection;
categoryCopy = categoryCopy ^ (categoryCopy & negativeNoise);
target ^= intersection;
if (categoryCopy.IsEmpty)
{
categoriesWithNegativeAssertions.Add(category, null);
}
else
{
categoriesWithNegativeAssertions.Add(category, categoryCopy);
}
}
}
categories = fullCategories.ToArray();
}
public static IEnumerable <RegularLanguageSet.SwitchPair <char, RegularLanguageSet.RangeSet> > GetInefficientSwitchCases(this RegularLanguageSet check)
{
return(check.GetRange().Where(r => r.Which == RegularLanguageSet.SwitchPairElement.CharacterRange));
}
private static Tuple <RegularLanguageSet, UnicodeCategory[], Dictionary <UnicodeCategory, RegularLanguageSet> > Breakdown(RegularLanguageSet check, RegularLanguageSet fullCheck)
{
//if (check.IsNegativeSet)
// return new Tuple<RegularLanguageSet, UnicodeCategory[], Dictionary<UnicodeCategory, RegularLanguageSet>>(check, new UnicodeCategory[0], new Dictionary<UnicodeCategory, RegularLanguageSet>());
var noise = fullCheck.RelativeComplement(check);
UnicodeCategory[] unicodeCategories;
Dictionary <UnicodeCategory, RegularLanguageSet> partialCategories = new Dictionary <UnicodeCategory, RegularLanguageSet>();
ParserCompilerExtensions.PropagateUnicodeCategoriesAndPartials(ref check, noise, out unicodeCategories, out partialCategories);
return(new Tuple <RegularLanguageSet, UnicodeCategory[], Dictionary <UnicodeCategory, RegularLanguageSet> >(check, unicodeCategories, partialCategories));
}
internal static void PropagateUnicodeCategories(ref RegularLanguageSet toStringArray, out UnicodeCategory[] toStringUnicodeCategories)
{
PropagateUnicodeCategories(ref toStringArray, true, out toStringUnicodeCategories);
}
public UnicodeTargetPartialCategory(UnicodeCategory targetedCategory, RegularLanguageSet negativeAssertion)
: base(targetedCategory)
{
this.NegativeAssertion = negativeAssertion;
}
