internal static void ObtainReplacements(ICaptureTokenStructure structure, Dictionary <ITokenSource, ICaptureTokenStructuralItem> result) { foreach (var element in structure.Keys) { ObtainReplacements(structure[element], result); } }
public void BuildNFA(IOilexerGrammarFile source) { if (this.Source is IOilexerGrammarTokenEofEntry) { return; } this.captureType = this.DetermineKind(); Dictionary <ITokenSource, ICaptureTokenStructuralItem> replacements = new Dictionary <ITokenSource, ICaptureTokenStructuralItem>(); if (captureType.Value != RegularCaptureType.Recognizer) { this.structure = TokenStructuralExtractionCore.BuildStructureFor(this, source); replacements = TokenStructuralExtractionCore.ObtainReplacements(this.structure); } else { replacements = new Dictionary <ITokenSource, ICaptureTokenStructuralItem>(); } this.nfaState = new RegularLanguageNFARootState(this); bool first = true; foreach (var expression in this.Branches.Cast <InlinedTokenExpression>()) { expression.BuildState(replacements); var expressionNFA = expression.NFAState; if (first) { bool isEdge = expressionNFA.IsEdge; first = false; nfaState.Union(expression.NFAState); if (nfaState.IsEdge && !isEdge) { nfaState.IsEdge = isEdge; } } else { nfaState.Union(expression.NFAState); } } if (ForcedRecognizer) { List <RegularLanguageNFAState> flatline = new List <RegularLanguageNFAState>(); RegularLanguageNFAState.FlatlineState(this.nfaState, flatline); foreach (var state in flatline) { state.IgnoreSources = true; } } }
public ICaptureTokenStructure Union(ICaptureTokenStructure second) { if (this.Count == 0) { return(second); } else if (second.Count == 0) { return(this); } var tokenGrpSource = (ITokenGroupItem)second.Sources.Where(k => k is ITokenGroupItem).FirstOrDefault(); var result = new CaptureTokenStructure(); var left = this.Keys.Except(second.Keys).ToArray(); var middle = this.Keys.Intersect(second.Keys).ToArray(); var right = second.Keys.Except(this.Keys).ToArray(); foreach (var element in left) { result._Add(element, this[element]); } foreach (var element in middle) { var leftElement = this[element]; var rightElement = second[element]; var middleElement = leftElement.Union(rightElement); result._Add(element, middleElement); } foreach (var element in right) { var rStructure = second[element]; result._Add(element, rStructure); } if (this.ResultType == second.ResultType) { result.ResultType = this.ResultType; } else { result.ResultType = ResultedDataType.ComplexType; } result.ResultedTypeName = this.ResultedTypeName ?? second.ResultedTypeName; ((ControlledCollection <ITokenSource>)(result.Sources)).AddRange(this.Sources.ToArray()); ((ControlledCollection <ITokenSource>)(result.Sources)).AddRange(second.Sources.ToArray()); result.Optional = this.Optional || second.Optional; return(result); }
private static ICaptureTokenStructure BuildStructureFor(InlinedTokenEntry entry, ITokenExpressionSeries expressionSeries, IOilexerGrammarFile source) { ICaptureTokenStructure result = null; HashList <HashList <string> > currentResultVariants = new HashList <HashList <string> >(); foreach (var expression in expressionSeries) { var current = BuildStructureFor(entry, expressionSeries, expression, source); if (result == null) { result = current; } else { result = result.Union(current); } if (entry.CaptureKind == RegularCaptureType.Capturer) { var dataSet = new HashList <string>(current.Keys); if (!currentResultVariants.Any(k => k.SequenceEqual(dataSet))) { currentResultVariants.Add(dataSet); } } } foreach (var variant in currentResultVariants) { result.Structures.Add(variant); } if (expressionSeries == entry.Branches) { ((ControlledCollection <ITokenSource>)result.Sources).baseList.Add(entry); } result.ResultedTypeName = string.Format("{0}{1}{2}", source.Options.TokenPrefix, entry.Name, source.Options.TokenSuffix); return(result); }
private static void BuildStructureConstructors(ICaptureTokenStructure targetStructure, InlinedTokenEntry structureRoot, ITypeIdentityManager identityManager) { int stateIndex = 0; foreach (var element in targetStructure.Values) { element.StateIndex = stateIndex++; } IEnumerable <IEnumerable <Tuple <ICaptureTokenStructuralItem, bool> > > validVariations = new IEnumerable <Tuple <ICaptureTokenStructuralItem, bool> > [0]; //List<List<string>> resultantParameterSets = new List<List<string>>(); var firstOrDefSeries = (ITokenExpressionSeries)targetStructure.Sources.FirstOrDefault(k => k is ITokenExpressionSeries); var firstOrDefToken = (IOilexerGrammarTokenEntry)targetStructure.Sources.FirstOrDefault(k => k is IOilexerGrammarTokenEntry); var optg = PickOptionalGroups(firstOrDefToken == null ? firstOrDefSeries : firstOrDefToken.Branches, targetStructure); foreach (var parameterSet in targetStructure.Structures) { var limitedSet = (from param in parameterSet join item in targetStructure.Keys on param equals item select new { Name = item, Item = targetStructure[item] }).ToArray(); var optionalSet = (from l in limitedSet where l.Item.Optional || l.Item.GroupOptional select l).ToArray(); if (limitedSet.Length > 0) { var parameterPermutations = VariateSeries(new LockedLinkedList <ICaptureTokenStructuralItem>(from l in limitedSet select l.Item).First, optg).Distinct(); validVariations = validVariations.Concat(parameterPermutations); foreach (var variation in parameterPermutations) { bool[] parameterStateFlags = (from k in variation.ToArray() select k.Item2).ToArray(); List <string> parameterResult = new List <string>(); for (int parameterIndex = 0; parameterIndex < parameterStateFlags.Length; parameterIndex++) { if (parameterStateFlags[parameterIndex]) { var limitedEntry = limitedSet[parameterIndex]; parameterResult.Add(limitedEntry.Name); } } //if (!resultantParameterSets.Any(k => k.SequenceEqual(parameterResult))) // resultantParameterSets.Add(parameterResult); } } } List <IEnumerable <Tuple <ICaptureTokenStructuralItem, bool> > > toRemove = new List <IEnumerable <Tuple <ICaptureTokenStructuralItem, bool> > >(); var exclusiveDistinctions = (from set in validVariations.Distinct().ToArray() let parameters = from parameter in set where parameter.Item2 select parameter.Item1 orderby string.Join(string.Empty, from k in parameters select k.BucketName) select new HashList <ICaptureTokenStructuralItem>(parameters)).Distinct(); var stateField = targetStructure.ResultClass.Fields.Add(new TypedName("state", identityManager.ObtainTypeReference(RuntimeCoreType.Int32))); var toStringMethod = targetStructure.ResultClass.Methods.Add(new TypedName("ToString", identityManager.ObtainTypeReference(RuntimeCoreType.String))); toStringMethod.AccessLevel = AccessLevelModifiers.Public; toStringMethod.IsOverride = true; var toStringStateSwitch = toStringMethod.Switch(stateField.GetReference()); toStringMethod.Return(identityManager.ObtainTypeReference(RuntimeCoreType.String).GetTypeExpression().GetField("Empty")); var formatMethod = identityManager.ObtainTypeReference(RuntimeCoreType.String).GetTypeExpression().GetMethod("Format"); foreach (var parameterSetVariation in exclusiveDistinctions) { TypedNameSeries currentCtorTNS = new TypedNameSeries(); int currentStateValue = 0; foreach (var parameterEntry in parameterSetVariation) { var parameterName = parameterEntry.BucketName; var currentElement = targetStructure[parameterName]; IType parameterType = null; switch (currentElement.ResultType) { case ResultedDataType.EnumerationItem: case ResultedDataType.Flag: parameterType = identityManager.ObtainTypeReference(RuntimeCoreType.Boolean); break; case ResultedDataType.Enumeration: ICaptureTokenStructure enumStructure = (ICaptureTokenStructure)currentElement; parameterType = enumStructure.AggregateSetEnum ?? enumStructure.ResultEnumSet[0]; break; case ResultedDataType.ComplexType: ICaptureTokenStructure dualStructure = (ICaptureTokenStructure)currentElement; parameterType = dualStructure.ResultInterface; break; case ResultedDataType.Counter: parameterType = identityManager.ObtainTypeReference(RuntimeCoreType.Int32); break; case ResultedDataType.Character: parameterType = identityManager.ObtainTypeReference(RuntimeCoreType.Char); break; case ResultedDataType.String: parameterType = identityManager.ObtainTypeReference(RuntimeCoreType.String); break; default: throw new InvalidOperationException("Unknown parameter type."); } currentStateValue |= (int)Math.Pow(2, parameterEntry.StateIndex); currentCtorTNS.Add(LowerFirstCharacter(parameterName), parameterType); } var currentCtor = targetStructure.ResultClass.Constructors.Add(currentCtorTNS); currentCtor.AccessLevel = AccessLevelModifiers.Public; currentCtor.Assign(stateField.GetReference(), currentStateValue.ToPrimitive()); var currentStateCase = toStringStateSwitch.Case(currentStateValue.ToPrimitive()); var currentInvocation = formatMethod.Invoke(); currentStateCase.Return(currentInvocation); var currentFormat = string.Empty.ToPrimitive(); currentInvocation.Arguments.Add(currentFormat); StringBuilder formatBuilder = new StringBuilder(); bool first = true; int currentParamIndex = 0; foreach (var parameterEntry in parameterSetVariation) { if (first) { first = false; } else { formatBuilder.Append(", "); } formatBuilder.AppendFormat("{{{0}}}", currentParamIndex++); var parameterName = parameterEntry.BucketName; var currentElement = targetStructure[parameterName]; IIntermediateFieldMember currentField = currentElement.AssociatedField; currentInvocation.Arguments.Add(currentField.GetReference()); currentCtor.Assign(currentField.GetReference(), currentCtor.Parameters[LowerFirstCharacter(parameterName)].GetReference()); } currentFormat.Value = formatBuilder.ToString(); } }
private static List <ICaptureTokenStructuralItem> PollGroup(ITokenExpressionSeries series, ICaptureTokenStructure structure) { List <ICaptureTokenStructuralItem> result = new List <ICaptureTokenStructuralItem>(); foreach (var expression in series) { foreach (var item in expression) { var firstOrDef = structure.Values.FirstOrDefault(k => k.Sources.Contains(item)); if (firstOrDef != null) { result.Add(firstOrDef); } else if (item is ITokenGroupItem) { result.AddRange(PollGroup((ITokenGroupItem)item, structure)); } } } return(result); }
private static Dictionary <ITokenGroupItem, List <ICaptureTokenStructuralItem> > PickOptionalGroups(ITokenExpressionSeries series, ICaptureTokenStructure structure) { Dictionary <ITokenGroupItem, List <ICaptureTokenStructuralItem> > result = new Dictionary <ITokenGroupItem, List <ICaptureTokenStructuralItem> >(); foreach (var expression in series) { foreach (var item in expression) { if (item is ITokenGroupItem) { var groupItem = (ITokenGroupItem)item; if ((item.RepeatOptions.Options & ScannableEntryItemRepeatOptions.ZeroOrMore) == ScannableEntryItemRepeatOptions.ZeroOrMore || (item.RepeatOptions.Options & ScannableEntryItemRepeatOptions.ZeroOrOne) == ScannableEntryItemRepeatOptions.ZeroOrOne || (item.RepeatOptions.Options & ScannableEntryItemRepeatOptions.Specific) == ScannableEntryItemRepeatOptions.Specific && (!item.RepeatOptions.Min.HasValue || item.RepeatOptions.Min.Value == 0)) { /* * * Found an optional token group, now to search * within to see what we have inside. * */ var firstOrDef = structure.Values.FirstOrDefault(k => k.Sources.Contains(item)); if (firstOrDef != null) { result.Add(groupItem, new List <ICaptureTokenStructuralItem>() { firstOrDef }); } else { var currentSet = PollGroup(groupItem, structure); if (currentSet.Count > 0) { result.Add(groupItem, currentSet); } } } else { var subResult = PickOptionalGroups(groupItem, structure); foreach (var element in subResult.Keys) { if (result.ContainsKey(element)) { result[element].AddRange(subResult[element]); } } } } } } return(result); }
internal static SelectiveTuple <Tuple <IIntermediateClassType, IIntermediateInterfaceType>, IIntermediateEnumType, Tuple <IIntermediateEnumType, IIntermediateEnumType[]> > CreateProgramStructure(ICaptureTokenStructure targetStructure, IIntermediateNamespaceDeclaration owner, InlinedTokenEntry structureRoot, ParserCompiler compiler) { switch (targetStructure.ResultType) { case ResultedDataType.Enumeration: bool isFlagSet = targetStructure.Values.All(k => k.ResultType == ResultedDataType.FlagEnumerationItem); bool useBucketName = targetStructure != structureRoot.CaptureStructure; if (targetStructure.Count > sizeof(SlotType) * 8 && isFlagSet) { var chunkedSets = targetStructure.Values.ToArray().Chunk(sizeof(SlotType) * 8); IIntermediateEnumType[] resultSet = new IIntermediateEnumType[chunkedSets.Length]; IIntermediateEnumType aggregateSet = null; aggregateSet = owner.Parts.Add().Enums.Add("{0}Cases", targetStructure.ResultedTypeName); for (int chunkIndex = 0; chunkIndex < chunkedSets.Length; chunkIndex++) { var chunkSet = chunkedSets[chunkIndex]; int fieldIndex = 0; var resultEnum = resultSet[chunkIndex] = owner.Parts.Add().Enums.Add("Valid{0}Set{1}", useBucketName ? targetStructure.BucketName : structureRoot.Name, (chunkIndex + 1)); resultEnum.AccessLevel = AccessLevelModifiers.Public; resultEnum.Fields.Add("None"); #if x86 resultEnum.ValueType = EnumerationBaseType.UInt32; #elif x64 resultEnum.ValueType = EnumerationBaseType.UInt64; #endif foreach (var element in chunkSet) { switch (element.ResultType) { case ResultedDataType.EnumerationItem: resultEnum.Fields.Add(element.BucketName); break; case ResultedDataType.FlagEnumerationItem: var resultField = resultEnum.Fields.Add(element.BucketName, (SlotType)Math.Pow(2, fieldIndex++)); break; default: throw new InvalidOperationException("Enums are supposed to yield an " + "enumeration, result element is of an invalid type."); } aggregateSet.Fields.Add(element.BucketName); } } targetStructure.ResultEnumSet = resultSet; targetStructure.AggregateSetEnum = aggregateSet; return(new SelectiveTuple <Tuple <IIntermediateClassType, IIntermediateInterfaceType>, IIntermediateEnumType, Tuple <IIntermediateEnumType, IIntermediateEnumType[]> >(Tuple.Create(aggregateSet, resultSet))); } else { IIntermediateEnumType resultEnum = null; IIntermediateEnumType aggregateSet = null; if (isFlagSet) { aggregateSet = owner.Parts.Add().Enums.Add("{0}Cases", targetStructure.ResultedTypeName); resultEnum = owner.Parts.Add().Enums.Add("Valid{0}", useBucketName ? targetStructure.BucketName : structureRoot.Name); } else { resultEnum = owner.Parts.Add().Enums.Add(targetStructure.ResultedTypeName); } resultEnum.AccessLevel = AccessLevelModifiers.Public; int numNonFlags = (from f in targetStructure.Values where f.ResultType == ResultedDataType.EnumerationItem select f).Count(); int flagFieldIndex = isFlagSet ? 0 : (int)(Math.Ceiling(Math.Log10(numNonFlags) / Math.Log10(2))); int regularFieldIndex = 1; #if x86 resultEnum.ValueType = EnumerationBaseType.UInt32; #elif x64 resultEnum.ValueType = EnumerationBaseType.UInt64; #endif resultEnum.Fields.Add("None", (SlotType)0); foreach (var element in targetStructure) { switch (element.Value.ResultType) { case ResultedDataType.EnumerationItem: resultEnum.Fields.Add(element.Value.BucketName, (SlotType)regularFieldIndex++); break; case ResultedDataType.FlagEnumerationItem: var resultField = resultEnum.Fields.Add(element.Value.BucketName, (SlotType)Math.Pow(2, flagFieldIndex++)); break; default: throw new InvalidOperationException("Enums are supposed to yield an " + "enumeration, result element is an of invalid type."); } if (isFlagSet) { aggregateSet.Fields.Add(element.Value.BucketName); } } targetStructure.ResultEnumSet = new IIntermediateEnumType[1] { resultEnum }; targetStructure.AggregateSetEnum = aggregateSet; return(new SelectiveTuple <Tuple <IIntermediateClassType, IIntermediateInterfaceType>, IIntermediateEnumType, Tuple <IIntermediateEnumType, IIntermediateEnumType[]> >(resultEnum)); } case ResultedDataType.ComplexType: var resultClass = owner.Parts.Add().Classes.Add(targetStructure.ResultedTypeName); var resultInterface = owner.Parts.Add().Interfaces.Add("I{0}", targetStructure.ResultedTypeName); bool tailLanguage = compiler.Source.Options.GrammarName.ToLower().EndsWith("language"); resultInterface.SummaryText = string.Format("Represents the {0} structure from the {1}{2}.", targetStructure.ResultedTypeName, compiler.Source.Options.GrammarName, tailLanguage ? string.Empty : " language"); resultClass.SummaryText = string.Format("Provides a base implementation of a @s:{2}; from the {1}{3}.", targetStructure.Name, compiler.Source.Options.GrammarName, resultInterface.Name, tailLanguage ? string.Empty : " language"); resultInterface.AccessLevel = AccessLevelModifiers.Public; resultClass.AccessLevel = AccessLevelModifiers.Internal; resultClass.ImplementedInterfaces.ImplementInterfaceQuick(resultInterface); targetStructure.ResultClass = resultClass; targetStructure.ResultInterface = resultInterface; foreach (var element in targetStructure.Values) { IType simpleType; switch (element.ResultType) { case ResultedDataType.Enumeration: case ResultedDataType.ComplexType: ICaptureTokenStructure cts = element as ICaptureTokenStructure; if (cts == null) { throw new InvalidOperationException("Complex types are supposed to be ICaptureTokenStructure instances."); } cts.ResultedTypeName = string.Format("{0}{1}", targetStructure.ResultedTypeName, cts.Name); var currentResult = CreateProgramStructure(cts, owner, structureRoot, compiler); currentResult.Visit( dualClassInterface => { var interfaceProp = resultInterface.Properties.Add(new TypedName(cts.BucketName, dualClassInterface.Item2), true, false); var classProp = resultClass.Properties.Add(new TypedName(cts.BucketName, dualClassInterface.Item2), true, false); classProp.AccessLevel = AccessLevelModifiers.Public; var classField = resultClass.Fields.Add(new TypedName("_{0}", dualClassInterface.Item2, LowerFirstCharacter(cts.BucketName))); classField.AccessLevel = AccessLevelModifiers.Private; classProp.GetMethod.Return(classField.GetReference()); cts.AssociatedField = classField; }, enumeration => { var interfaceProp = resultInterface.Properties.Add(new TypedName(cts.BucketName, enumeration), true, false); var classProp = resultClass.Properties.Add(new TypedName(cts.BucketName, enumeration), true, false); classProp.AccessLevel = AccessLevelModifiers.Public; var classField = resultClass.Fields.Add(new TypedName("_{0}", enumeration, LowerFirstCharacter(cts.BucketName))); classField.AccessLevel = AccessLevelModifiers.Private; classProp.GetMethod.Return(classField.GetReference()); cts.AssociatedField = classField; }, aggregateWithValidSet => { var enumeration = aggregateWithValidSet.Item1; var interfaceProp = resultInterface.Properties.Add(new TypedName(cts.BucketName, enumeration), true, false); var classProp = resultClass.Properties.Add(new TypedName(cts.BucketName, enumeration), true, false); classProp.AccessLevel = AccessLevelModifiers.Public; var classField = resultClass.Fields.Add(new TypedName("_{0}", enumeration, LowerFirstCharacter(cts.BucketName))); classField.AccessLevel = AccessLevelModifiers.Private; classProp.GetMethod.Return(classField.GetReference()); cts.AssociatedField = classField; }, () => { throw new InvalidOperationException("Complex types are supposed to yield a complex type, result type unknown"); }); break; case ResultedDataType.EnumerationItem: case ResultedDataType.Flag: simpleType = owner.IdentityManager.ObtainTypeReference(RuntimeCoreType.Boolean); goto simpleType; case ResultedDataType.Counter: simpleType = owner.IdentityManager.ObtainTypeReference(RuntimeCoreType.Int32); goto simpleType; case ResultedDataType.Character: simpleType = owner.IdentityManager.ObtainTypeReference(RuntimeCoreType.Char); goto simpleType; case ResultedDataType.String: simpleType = owner.IdentityManager.ObtainTypeReference(RuntimeCoreType.String); goto simpleType; default: break; } continue; simpleType: { var interfaceProp = resultInterface.Properties.Add(new TypedName(element.BucketName, simpleType), true, false); var classProp = resultClass.Properties.Add(new TypedName(element.BucketName, simpleType), true, false); classProp.AccessLevel = AccessLevelModifiers.Public; var classField = resultClass.Fields.Add(new TypedName("_{0}", simpleType, LowerFirstCharacter(element.BucketName))); classField.AccessLevel = AccessLevelModifiers.Private; classProp.GetMethod.Return(classField.GetReference()); element.AssociatedField = classField; } } BuildStructureConstructors(targetStructure, structureRoot, owner.IdentityManager); return(new SelectiveTuple <Tuple <IIntermediateClassType, IIntermediateInterfaceType>, IIntermediateEnumType, Tuple <IIntermediateEnumType, IIntermediateEnumType[]> >(Tuple.Create(resultClass, resultInterface))); case ResultedDataType.PassThrough: return(null); default: throw new InvalidOperationException(); } }
internal static Dictionary <ITokenSource, ICaptureTokenStructuralItem> ObtainReplacements(ICaptureTokenStructure structure) { Dictionary <ITokenSource, ICaptureTokenStructuralItem> result = new Dictionary <ITokenSource, ICaptureTokenStructuralItem>(); ObtainReplacements(structure, result); return(result); }