public PNode parseTokensIntoPNodes(ProseRuntime runtime, ProseClient who, List <LexerToken> tokens) { // Save the parameters this.who = who; this.runtime = runtime; sourceTokenList = tokens; setupForParsing(); Trie <RawWord, Word> .Node currWordLookupNode, wordLookupRoot, lastGoodNode; wordLookupRoot = runtime.getWordLookupRoot(); currWordLookupNode = wordLookupRoot; lastGoodNode = null; int lastGoodNodeTokenIdx = -1; // Record whether or not we're in the process of building up a word. bool isBuildingWord = false; int lastProcessedTokenIdx = -1; while (tokenIdx < tokens.Count) { LexerToken token = tokens[tokenIdx]; // // Deal With Quadquotes. // if (token.rawWord == ProseLanguage.Raw.Quadquote) { // First clean up any word we may be building and write it to output. if (isBuildingWord) { if (lastGoodNode == null) { // // If we're inside a quadquote block then this is fine: even if we don't have // // a legitimate word we can still wrap rawwords. Otherwise it's an error. // if (insideQuadquoteExpression) { // If there is no last good match, then take the raw words we've passed // and dump them all into raw word objects. for (int i = lastProcessedTokenIdx + 1; i < tokenIdx; i++) { writePNode(new PNode(new RawWordObject(tokens[i].rawWord))); } // Update everything so we continue after this point lastGoodNodeTokenIdx = tokenIdx - 1; lastProcessedTokenIdx = lastGoodNodeTokenIdx; currWordLookupNode = wordLookupRoot; isBuildingWord = false; // Don't bother updating tokenIdx because we need to look at the word again. } // else { // throw new RuntimeLexerSourceException("Unrecognized word or symbol.", tokens[tokenIdx-1]); // } } else { writePNode(new PNode(lastGoodNode.Value)); // Reset everything so we're looking for a new word again. lastGoodNode = null; currWordLookupNode = wordLookupRoot; isBuildingWord = false; lastGoodNodeTokenIdx = tokenIdx; lastProcessedTokenIdx = lastGoodNodeTokenIdx; } } // Output a quadquote writePNode(new PNode(runtime.Quadquote)); // Toggle our quad-quote-state. insideQuadquoteExpression = !insideQuadquoteExpression; lastQuadquoteIdx = tokenIdx; lastProcessedTokenIdx = tokenIdx; // Continue tokenIdx++; continue; } if (insideQuadquoteExpression) { if (token.tokenType != LexerToken.TYPE.UNCLASSIFIED) { throw new RuntimeLexerFailure("Static Lexer Failed Token Classification."); } // // This code is essentially copied from the LexerToken.TYPE.UNCLASSIFIED block below. // The only major difference is that instead of throwing an exception we wrap unknown // text inside raw word objects. // isBuildingWord = true; // Try to continue the current word matching. Trie <RawWord, Word> .Node nodeForThisRawWord = currWordLookupNode.getChildNode(token.rawWord); // If we can't continue this way... if (nodeForThisRawWord == null) { //...then whatever our last good match was is the correct word. if (lastGoodNode == null) { // If there is no last good match, then take the raw words we've passed // and dump them all into raw word objects. for (int i = lastProcessedTokenIdx + 1; i < tokenIdx; i++) { writePNode(new PNode(new RawWordObject(tokens[i].rawWord))); } lastProcessedTokenIdx = tokenIdx - 1; // Update everything so we continue after this point // Don't bother updating tokenIdx because we need to look at the word again. // Do update currWordLookupNode currWordLookupNode = wordLookupRoot.getChildNode(token.rawWord); // If there's no node at all, we have to deal with it now if (currWordLookupNode == null) { writePNode(new PNode(new RawWordObject(token.rawWord))); lastGoodNodeTokenIdx = tokenIdx; isBuildingWord = false; currWordLookupNode = wordLookupRoot; lastProcessedTokenIdx = tokenIdx; tokenIdx++; } else { isBuildingWord = true; lastGoodNodeTokenIdx = tokenIdx - 1; tokenIdx++; } continue; } writePNode(new PNode(lastGoodNode.Value)); // Reset everything so we're looking for a new word again. lastGoodNode = null; currWordLookupNode = wordLookupRoot; isBuildingWord = false; lastProcessedTokenIdx = lastGoodNodeTokenIdx; // Move the head back to the spot after the last token in the word tokenIdx = lastGoodNodeTokenIdx + 1; continue; } // If adding this raw word makes a word, then record it as good if (nodeForThisRawWord.Value != null) { lastGoodNode = nodeForThisRawWord; lastGoodNodeTokenIdx = tokenIdx; } currWordLookupNode = nodeForThisRawWord; continue; } else { switch (token.tokenType) { case LexerToken.TYPE.UNCLASSIFIED: { isBuildingWord = true; // Try to continue the current word matching. Trie <RawWord, Word> .Node nodeForThisRawWord = currWordLookupNode.getChildNode(token.rawWord); // If we can't continue this way... if (nodeForThisRawWord == null) { //...then whatever our last good match was is the correct word. if (lastGoodNode == null) { //throw new RuntimeLexerSourceException("Unrecognized word or symbol.", token); // Dump everything into raw words. tokenIdx++; // Include this word for (int i = lastProcessedTokenIdx + 1; i < tokenIdx; i++) { writePNode(new PNode(new RawWordObject(tokens[i].rawWord))); } // Update everything so we continue after this point lastGoodNodeTokenIdx = tokenIdx - 1; lastProcessedTokenIdx = lastGoodNodeTokenIdx; currWordLookupNode = wordLookupRoot; isBuildingWord = false; continue; } writePNode(new PNode(lastGoodNode.Value)); // Reset everything so we're looking for a new word again. lastGoodNode = null; currWordLookupNode = wordLookupRoot; isBuildingWord = false; lastProcessedTokenIdx = lastGoodNodeTokenIdx; // Move the head back to the spot after the last token in the word tokenIdx = lastGoodNodeTokenIdx + 1; continue; } // If adding this raw word makes a word, then record it as good if (nodeForThisRawWord.Value != null) { lastGoodNode = nodeForThisRawWord; lastGoodNodeTokenIdx = tokenIdx; } currWordLookupNode = nodeForThisRawWord; continue; } break; case LexerToken.TYPE.STRING: { // First clean up any word we may be building and write it to output. if (isBuildingWord) { if (lastGoodNode == null || lastGoodNodeTokenIdx != tokenIdx - 1) { //throw new RuntimeLexerSourceException("Unrecognized word or symbol.", tokens[tokenIdx-1]); // Just take all the words up until now and dump them into raw words. for (int i = lastProcessedTokenIdx + 1; i < tokenIdx; i++) { writePNode(new PNode(new RawWordObject(tokens[i].rawWord))); } // Update everything so we continue after this point lastGoodNodeTokenIdx = tokenIdx - 1; lastProcessedTokenIdx = lastGoodNodeTokenIdx; currWordLookupNode = wordLookupRoot; isBuildingWord = false; } else { writePNode(new PNode(lastGoodNode.Value)); // Reset everything so we're looking for a new word again. lastGoodNode = null; currWordLookupNode = wordLookupRoot; isBuildingWord = false; lastProcessedTokenIdx = lastGoodNodeTokenIdx; } } // Now write the string literal object to output writePNode(new PNode(new StringLiteralObject(token.rawWord.AsString))); lastProcessedTokenIdx = tokenIdx; // Continue tokenIdx++; continue; } break; } } } finalCheckAfterParsing(); return(outputRoot); }
private void while_MATCHING_PATTERN_extendWith(PNode node, Trie <ProseObject, List <Phrase> > .Node patternNode) { // If this is the first node we're pushing in as a "pattern", then add it to patternComponentNodes if (currPatternObject.Length == 0) { patternComponentNodes.Add(node); } ProseObject obj = node.value; switch (bracketReaderCount) { case 0: if (obj == runtime.LeftSquareBracket) { // Attempting to name an argument when no argument type preceeds it! if (currPatternObject.Length == 0) { state = MatcherState.FAILED; return; } numObjectsMatched++; setBracketReaderCount(1); } else { if (obj is Word) { currPatternObject.putPatternElement(obj); numObjectsMatched++; } else { state = MatcherState.FAILED; return; } } break; case 1: if (obj == runtime.RightSquareBracket) { justPutArgName = false; setBracketReaderCount(0); numObjectsMatched++; } // If its a word or raw word then it should be an argument name else if (obj is Word || obj is RawWord) { if (!justPutArgName) { currPatternObject.replaceLastPutElementNameWith(obj); justPutArgName = true; numObjectsMatched++; } else { // Can't supply two arg names. state = MatcherState.FAILED; return; } } break; } // VERY IMPORTANT: Keep the matcher moving up the tree. currNode = patternNode; }
List <PatternMatcher> while_MATCHING_PATTERN_matchNextObject(PNode node) { List <PatternMatcher> babyMatchers = new List <PatternMatcher>(); ProseObject obj = node.value; // @Pattern won't match periods if (obj == runtime.Period) { return(babyMatchers); } // switch (bracketReaderCount) { case 0: // Check to see if obj is -> :+ or :- if (obj == runtime.RightArrow || obj == runtime.ColonPlus || obj == runtime.ColonMinus) { // If so, then we have to leave matching. See if anything is checking for this symbol Trie <ProseObject, List <Phrase> > .Node childNode = currNode.getChildNode(obj); if (childNode != null) { PatternMatcher babyMatcher = makeCopyWithStateFromAtWord(obj); // Clone ourselves babyMatcher.while_MATCHING_OBJECT_extendWith(node, childNode); babyMatchers.Add(babyMatcher); } } else if (obj == runtime.Period || !(obj is Word)) { return(babyMatchers); } else { PatternMatcher babyMatcher = makeCopyWithStateFromAtWord(runtime.@pattern); babyMatcher.while_MATCHING_PATTERN_extendWith(node, currNode); // Use same node if (babyMatcher.IsntFailed) { babyMatchers.Add(babyMatcher); } } break; case 1: { // In this case, just trust while_..._extendWith to do the right thing. PatternMatcher babyMatcher = makeCopyWithStateFromAtWord(runtime.@pattern); babyMatcher.bracketReaderCount = bracketReaderCount; babyMatcher.while_MATCHING_PATTERN_extendWith(node, currNode); // Use same node if (babyMatcher.IsntFailed) { babyMatchers.Add(babyMatcher); } break; } } return(babyMatchers); }
List <PatternMatcher> while_MATCHING_PROSE_matchNextObject(PNode node) { ProseObject obj = node.value; // Return this list of spawned matchers. List <PatternMatcher> babyMatchers = new List <PatternMatcher>(); // Possibly we can make this prose block longer (in addition to other options where we may leave it) bool canExtendThisProseBlock = false; // If we want extend our prose block by adding a right parenthetical, remember to pop the left off the stack. bool shouldPopParentheticalStack = false; // If we stay in this prose block, record if we are entering text or leaving text bool shouldToggleInText = false; // Should we exit to a different mode? bool shouldLeaveMatchingProse = false; MatcherState switchToThisState; // Check for parentheticals // No need to check for () because it's eliminated before we're ever called. if (obj == runtime.LeftCurlyBracket) { canExtendThisProseBlock = true; // Do this in extend //parentheticalStack.Push(runtime.LeftCurlyBracket); } else if (obj == runtime.LeftSquareBracket) { canExtendThisProseBlock = true; // Do this in extend //parentheticalStack.Push(runtime.LeftSquareBracket); } else if (obj == runtime.RightCurlyBracket) { ProseObject matchingParenthetical = tryPeekParenthetical(); if (matchingParenthetical == null || matchingParenthetical != runtime.LeftCurlyBracket) { throw new PatternMatcherException("Right curly bracket wihtout matching left curly bracket inside @prose block.", this); } // At this point, we agree the curly bracket makes sense, so we pop it's counterpart off the stack. shouldPopParentheticalStack = true; canExtendThisProseBlock = true; } else if (obj == runtime.RightSquareBracket) { ProseObject matchingParenthetical = tryPeekParenthetical(); if (matchingParenthetical == null || matchingParenthetical != runtime.LeftSquareBracket) { throw new PatternMatcherException("Right square bracket without matching left square bracket inside @prose block.", this); } // At this point, we agree the square bracket makes sense, so we pop it's counterpart off the stack. shouldPopParentheticalStack = true; canExtendThisProseBlock = true; } else if (obj == runtime.Quadquote) { // In this case, we count that we are in a text block, but we don't "read it as text". Instead, we just // bundle the symbols in with our prose object and pass it along to be interpreted later. if (inText) { ProseObject matchingParenthetical = tryPeekParenthetical(); if (matchingParenthetical == null || matchingParenthetical != runtime.Quadquote) { // The "" didn't match up with a previous one! throw new PatternMatcherException("Quadquote entagled with other parenthetical.", this); } else { canExtendThisProseBlock = true; shouldToggleInText = true; // inText = false; shouldPopParentheticalStack = true; // Pop off the left "" } } else { // In this case, we treat this as an opening quadquote. canExtendThisProseBlock = true; parentheticalStack.Push(runtime.Quadquote); // Remember the left "" shouldToggleInText = true; // inText = true; } } // When the parenthetical stack is empty, we are allowed to exit the prose block if // we can match the next piece of the pattern. else if (parentheticalStack.Count == 0) { // if (obj == runtime.Period) { // // A period ends a "prose" block unless it is protected by {} or [] or "" "" // // The period does not appear in the block. It is interpreted as ending the sentence. // // So exit reading prose // shouldLeaveMatchingProse = true; // //switchToThisState = MatcherState.MATCHING_OBJECT; // //CURSOR // } else { // Use inheritance to look for an exit! // Look up words which would match if they were in a pattern. List <ProseObject> matchingPatternWords = getMatchingPatternWords(obj); // Look these words up to see if actual patterns exist. canExtendThisProseBlock = (matchingPatternWords.Count != 0); foreach (ProseObject match in matchingPatternWords) { // If it's not a period and the attempt to extend @prose immediately failed, // then we continue matching @prose. If the attempt didn't immediately fail then // the rules say we must leave @prose. if (obj == runtime.Period) // && !babyMatcher.IsntFailed) { canExtendThisProseBlock = false; } if (match == runtime.@prose) { continue; // back to back @prose outlawed. } // Look up the match Trie <ProseObject, List <Phrase> > .Node matchNode = currNode.getChildNode(match); if (matchNode == null) { continue; } // If this pattern can be extended by leaving prose, then do it // Spawn a baby matcher to pursue this pattern PatternMatcher babyMatcher = makeCopyWithStateFromAtWord(match); // Clone ourselves babyMatcher.switchToState_MATCHING_OBJECT(); // Move babyMatcher into the generic state (can accept anything) babyMatcher.while_MATCHING_OBJECT_extendWith(node, matchNode); // Append the new node babyMatchers.Add(babyMatcher); // Eventually return this baby matcher // If it's possible to exit an @prose block, the matcher MUST canExtendThisProseBlock = false; } } // In this case the parenthetical stack is not empty and we have some symbol that // isn't parenthetical (or ""). else { // Add it to this prose block canExtendThisProseBlock = true; } // // Perform updates according to flags // if (shouldToggleInText) { inText = !inText; } if (shouldPopParentheticalStack) { parentheticalStack.Pop(); } if (canExtendThisProseBlock) { // Extend ourselves! // NOTE: We don't change the node we're using! PatternMatcher babyMatcher = makeCopyWithStateFromAtWord(runtime.@prose); // Clone ourselves babyMatcher.objectsInCurrentProseblock = objectsInCurrentProseblock; babyMatcher.while_MATCHING_PROSE_extendWith(node, currNode); babyMatchers.Add(babyMatcher); } else if (shouldLeaveMatchingProse) { throw new Exception("Failboat."); // PatternMatcher babyMatcher = makeCopyWithStateFromAtWord(obj); // Clone ourselves // //babyMatcher.switchToState_MATCHING_OBJECT(); // babyMatcher.while_MATCHING_OBJECT_extendWith(node, ); // babyMatchers.Add(babyMatcher); } return(babyMatchers); }
// Read a new object and maybe switch to reading prose, text, or pattern if necessary. List <PatternMatcher> while_MATCHING_OBJECT_matchNextObject(PNode node) { ProseObject obj = node.value; // Some symbols get special treatment at the beginning or end of if (obj == runtime.Period || obj == runtime.Semicolon && currNode == patternTrie.Root) { } // Get a list of words which, if they appeared in a pattern, would match with this one. List <ProseObject> matchingPatternWords = getMatchingPatternWords(obj); // // For each matchingPatternWord, query the pattern tree to see if // our pattern can be extended any further. // List <PatternMatcher> babyMatchers = new List <PatternMatcher>(16); bool foundAMatch = false; //ProseObject myExtensionObject = null; Trie <ProseObject, List <Phrase> > .Node myExtensionNode = null; foreach (ProseObject objMatch in matchingPatternWords) { ProseObject match = objMatch; //Word match = (Word) objMatch; // Look up the node that would correspond to the pattern word that would match. Trie <ProseObject, List <Phrase> > .Node matchNode = currNode.getChildNode(match); // If we can't find it, then there are no patterns with that matching word, so skip it if (matchNode == null) { continue; } // If we can find it, then we might need to make a new matcher // Since ONE match has to be reserved for THIS, and the other matches spawn new matchers, // we use "foundAMatch" to decide which to do. // if (foundAMatch) // { // Spawn off a baby matcher and transform its state to represent the new possibility. PatternMatcher babyMatcher = this.makeCopyWithStateFromAtWord(match); babyMatcher.while_MATCHING_OBJECT_extendWith(node, matchNode); babyMatchers.Add(babyMatcher); // List this baby matcher foundAMatch = true; // } // else { // // Cache these two so we can use them later. // myExtensionObject = match; // myExtensionNode = matchNode; // foundAMatch = true; // } } // If we found at least one, then we need to tweak ourselves and add ourselves to the babies. // if (foundAMatch) { // this.while_MATCHING_OBJECT_extendWith(node, myExtensionNode); // babyMatchers.Add(this); // } // else { // state = MatcherState.FAILED; // } if (!foundAMatch) { state = MatcherState.FAILED; } return(babyMatchers); }
static public void constructInitialPatternTrie(ProseRuntime runtime) { ProseScope scope = runtime.GlobalScope; Trie <ProseObject, List <Phrase> > patternTrie = scope.PatternTree; // // Pattern Creation Pattern // // The only pattern we need to "force" into the system is the pattern for making patterns: // word[phrase_class] : @pattern[pattern] -> @prose[value] . // Phrase phrasePhrase = new SimplePhrase(runtime.Word_phrase, // new ProseObject[] { runtime.Word_word, //patternTrie.putObjectString( #region Word binding phrases // , word : @raw , { ProseObject[] commaDelimitedBindWordsPattern = new ProseObject[] { runtime.Comma, runtime.Word_word, runtime.Colon, runtime.@raw, runtime.Comma }; WordBindingPhrase bindWords_commaDelimited = new WordBindingPhrase(runtime.Word_phrase, commaDelimitedBindWordsPattern); scope.addPhrase(bindWords_commaDelimited); } // , word +: @raw , { ProseObject[] commaDelimitedBindWordsPattern = new ProseObject[] { runtime.Comma, runtime.Word_word, runtime.PlusColon, runtime.@raw, runtime.Comma }; WordBindingPhrase bindWords_commaDelimited = new WordBindingPhrase(runtime.Word_phrase, commaDelimitedBindWordsPattern); scope.addPhrase(bindWords_commaDelimited); } // , word <- @raw , { ProseObject[] commaDelimitedBindWordsPattern = new ProseObject[] { runtime.Comma, runtime.Word_word, runtime.LeftArrow, runtime.@raw, runtime.Comma }; ExclusiveWordBindingPhrase exclusiveBindWords_commaDelimited = new ExclusiveWordBindingPhrase(runtime.Word_phrase, commaDelimitedBindWordsPattern); scope.addPhrase(exclusiveBindWords_commaDelimited); } #endregion #region Phrase creation phrases // Comma delimited exclusive phrase creation // , word[class]: @pattern -> @prose[value] , { ProseObject[] phrasePattern = new ProseObject[] { runtime.Comma, runtime.Word_word, runtime.Colon, runtime.@pattern, runtime.RightArrow, runtime.@prose, runtime.Comma }; ExclusivePhraseBindingPhrase phrasePhrase = new ExclusivePhraseBindingPhrase(runtime.Word_phrase, phrasePattern); scope.addPhrase(phrasePhrase); } // Semicolon delimited exclusive phrase creation // ; word[class]: @pattern -> @prose[value] ; { ProseObject[] phrasePattern = new ProseObject[] { runtime.Semicolon, runtime.Word_word, runtime.Colon, runtime.@pattern, runtime.RightArrow, runtime.@prose, runtime.Semicolon }; ExclusivePhraseBindingPhrase phrasePhrase = new ExclusivePhraseBindingPhrase(runtime.Word_phrase, phrasePattern); scope.addPhrase(phrasePhrase); } // Period delimited exclusive phrase creation // . word[class]: @pattern -> @prose[value] . { ProseObject[] phrasePattern = new ProseObject[] { runtime.Period, runtime.Word_word, runtime.Colon, runtime.@pattern, runtime.RightArrow, runtime.@prose, runtime.Period }; ExclusivePhraseBindingPhrase phrasePhrase = new ExclusivePhraseBindingPhrase(runtime.Word_phrase, phrasePattern); scope.addPhrase(phrasePhrase); } #endregion #region Reading // , read @string[x] , { ProseObject[] p = new ProseObject[] { runtime.Comma, runtime.word("read"), runtime.word("@string"), runtime.Comma }; Phrase readPhrase = new ReadPhrase(runtime.Word_phrase, p); scope.addPhrase(readPhrase); } // contents of text file @string[file_name] { ProseObject[] p = new ProseObject[] { runtime.word("contents"), runtime.word("of"), runtime.word("text"), runtime.word("file"), runtime.word("@string") }; Phrase readFilePhrase = new ContentsOfTextFilePhrase(runtime.Word_phrase, p); scope.addPhrase(readFilePhrase); } runtime.read("phrase: , read file @string[path] , -> , read contents of text file path ,", runtime.GlobalClient); #endregion #region Foreign Function Interface // Load an assembly and bind it to a name // , load assembly : @string[file_name] <- @raw[new_assembly_word] , { ProseObject[] p = new ProseObject[] { runtime.Comma, runtime.word("load"), runtime.word("assembly"), runtime.Colon, runtime.@string, runtime.LeftArrow, runtime.@raw, runtime.Comma }; BindAssemblyPhrase asmPhrase = new BindAssemblyPhrase(runtime.Word_phrase, p); scope.addPhrase(asmPhrase); } // Load a type and bind it to a name // , @assembly[asm_name] type : @string[type_name] <- @raw[new_type_word] , { ProseObject[] p = new ProseObject[] { runtime.Comma, runtime.word("@assembly"), runtime.word("type"), runtime.Colon, runtime.@string, runtime.LeftArrow, runtime.@raw, runtime.Comma }; BindTypePhrase typePhrase = new BindTypePhrase(runtime.Word_phrase, p); scope.addPhrase(typePhrase); } // Load a method and bind it to a name // , @type[type_name] method : @string[method_name] <- @raw[new_method_word] , { ProseObject[] p = new ProseObject[] { runtime.Comma, runtime.word("@type"), runtime.word("method"), runtime.Colon, runtime.@string, runtime.LeftArrow, runtime.@raw, runtime.Comma }; BindMethodPhrase methodPhrase = new BindMethodPhrase(runtime.Word_phrase, p); scope.addPhrase(methodPhrase); } // Apply a method to some arguments to produce an action // , @method[method_name] @prose[args] , { ProseObject[] p = new ProseObject[] { runtime.Comma, runtime.word("@method"), runtime.@prose, runtime.Comma }; Phrase applyMethodPhrase = new ApplyMethodPhrase(runtime.Word_phrase, p); scope.addPhrase(applyMethodPhrase); } // Apply a method with no arguments to produce an action // , @method[method_name] , { ProseObject[] p = new ProseObject[] { runtime.Comma, runtime.word("@method"), runtime.Comma }; Phrase applyMethodPhrase = new ApplyMethodPhrase(runtime.Word_phrase, p); scope.addPhrase(applyMethodPhrase); } #endregion // Add a breakpoint { ProseObject[] p = new ProseObject[] { runtime.@break }; Phrase addBreakpointPhrase = new BreakPointPhrase(runtime.Word_phrase, p); scope.addPhrase(addBreakpointPhrase); } #region Debugger #endregion #region Experimental // , -> @pattern -> , { ProseObject[] test = new ProseObject[] { runtime.Comma, runtime.RightArrow, runtime.@pattern, runtime.RightArrow, runtime.Comma }; DebugOutputPhrase dbg = new DebugOutputPhrase("Carlybou", runtime.Word_phrase, test); scope.addPhrase(dbg); } #endregion }