예제 #1
0
        public void read(List <ProseObject> source, int startIdx, int endIdx, ProseClient who)
        {
            //	Convert source into a linked list structure.
            PNode head       = new PNode();
            PNode initPeriod = new PNode(Period);

            head.next       = initPeriod;
            initPeriod.prev = head;
            PNode prev = initPeriod;

            for (int i = startIdx; i < endIdx; i++)
            {
                ProseObject obj = source[i];
                PNode       p   = new PNode(obj);
                p.prev    = prev;
                prev.next = p;
                // Update
                prev = p;
            }

            PNode termPeriod = new PNode(Period);

            prev.next       = termPeriod;
            termPeriod.prev = prev;

            //	Read the new format starting with the initial period.
            read(head);
        }
예제 #2
0
        public void read(string source, ProseClient who)
        {
            //	Insert a leading period because the parser expects the previous terminal symbol.
            source = "." + source + ".";

            //	Apply lexical analysis that requires no knowledge of the runtime words/phrases
            List <LexerToken> statLexSrc = staticLexer.parseStringIntoTokens(source);
            //	Parse LexerTokens into runtime code.
            PNode parsedSource = parseTokensIntoPNodes(global_client, statLexSrc);

            //	Debug
            //Console.WriteLine("After runtime lexer: " + parsedSource.getReadableString() );

            read(parsedSource);
        }
예제 #3
0
 //	Synchronize access to the runtime lexer
 private PNode parseTokensIntoPNodes(ProseClient client, List <LexerToken> statLexSrc)
 {
     lock (sentenceReaderLock) {
         return(runtimeLexer.parseTokensIntoPNodes(this, global_client, statLexSrc));
     }
 }
예제 #4
0
 //    Synchronize access to the runtime lexer
 private PNode parseTokensIntoPNodes(ProseClient client, List<LexerToken> statLexSrc)
 {
     lock(sentenceReaderLock) {
         return runtimeLexer.parseTokensIntoPNodes(this, global_client, statLexSrc);
     }
 }
예제 #5
0
        public void read(List<ProseObject> source, int startIdx, int endIdx, ProseClient who)
        {
            //	Convert source into a linked list structure.
            PNode head = new PNode();
            PNode initPeriod = new PNode(Period);
            head.next = initPeriod;
            initPeriod.prev = head;
            PNode prev = initPeriod;
            for (int i=startIdx; i < endIdx; i++) {
                ProseObject obj = source[i];
                PNode p = new PNode(obj);
                p.prev = prev;
                prev.next = p;
                // Update
                prev = p;
            }

            PNode termPeriod = new PNode(Period);
            prev.next = termPeriod;
            termPeriod.prev = prev;

            //	Read the new format starting with the initial period.
            read(head);
        }
예제 #6
0
        public void read(string source, ProseClient who)
        {
            //	Insert a leading period because the parser expects the previous terminal symbol.
            source = "." + source + ".";

            //	Apply lexical analysis that requires no knowledge of the runtime words/phrases
            List<LexerToken> statLexSrc = staticLexer.parseStringIntoTokens(source);
            //	Parse LexerTokens into runtime code.
            PNode parsedSource = parseTokensIntoPNodes(global_client, statLexSrc);

            //	Debug
            //Console.WriteLine("After runtime lexer: " + parsedSource.getReadableString() );

            read(parsedSource);
        }
예제 #7
0
        public PNode parseTokensIntoPNodes(ProseRuntime runtime, ProseClient who, List <LexerToken> tokens)
        {
            //	Save the parameters
            this.who        = who;
            this.runtime    = runtime;
            sourceTokenList = tokens;

            setupForParsing();

            Trie <RawWord, Word> .Node currWordLookupNode, wordLookupRoot, lastGoodNode;
            wordLookupRoot     = runtime.getWordLookupRoot();
            currWordLookupNode = wordLookupRoot;
            lastGoodNode       = null;
            int lastGoodNodeTokenIdx = -1;
            //	Record whether or not we're in the process of building up a word.
            bool isBuildingWord        = false;
            int  lastProcessedTokenIdx = -1;

            while (tokenIdx < tokens.Count)
            {
                LexerToken token = tokens[tokenIdx];

                //
                //	Deal With Quadquotes.
                //
                if (token.rawWord == ProseLanguage.Raw.Quadquote)
                {
                    //	First clean up any word we may be building and write it to output.
                    if (isBuildingWord)
                    {
                        if (lastGoodNode == null)
                        {
//							//	If we're inside a quadquote block then this is fine: even if we don't have
//							//	a legitimate word we can still wrap rawwords.  Otherwise it's an error.
//							if (insideQuadquoteExpression)
                            {
                                //	If there is no last good match, then take the raw words we've passed
                                //	and dump them all into raw word objects.
                                for (int i = lastProcessedTokenIdx + 1; i < tokenIdx; i++)
                                {
                                    writePNode(new PNode(new RawWordObject(tokens[i].rawWord)));
                                }
                                //	Update everything so we continue after this point
                                lastGoodNodeTokenIdx  = tokenIdx - 1;
                                lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                                currWordLookupNode    = wordLookupRoot;
                                isBuildingWord        = false;
                                //	Don't bother updating tokenIdx because we need to look at the word again.
                            }
//							else {
//								throw new RuntimeLexerSourceException("Unrecognized word or symbol.", tokens[tokenIdx-1]);
//							}
                        }
                        else
                        {
                            writePNode(new PNode(lastGoodNode.Value));
                            //	Reset everything so we're looking for a new word again.
                            lastGoodNode          = null;
                            currWordLookupNode    = wordLookupRoot;
                            isBuildingWord        = false;
                            lastGoodNodeTokenIdx  = tokenIdx;
                            lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                        }
                    }

                    //	Output a quadquote
                    writePNode(new PNode(runtime.Quadquote));
                    //	Toggle our quad-quote-state.
                    insideQuadquoteExpression = !insideQuadquoteExpression;
                    lastQuadquoteIdx          = tokenIdx;
                    lastProcessedTokenIdx     = tokenIdx;
                    //	Continue
                    tokenIdx++;
                    continue;
                }



                if (insideQuadquoteExpression)
                {
                    if (token.tokenType != LexerToken.TYPE.UNCLASSIFIED)
                    {
                        throw new RuntimeLexerFailure("Static Lexer Failed Token Classification.");
                    }

                    //
                    //	This code is essentially copied from the LexerToken.TYPE.UNCLASSIFIED block below.
                    //	The only major difference is that instead of throwing an exception we wrap unknown
                    //	text inside raw word objects.
                    //
                    isBuildingWord = true;
                    //	Try to continue the current word matching.
                    Trie <RawWord, Word> .Node nodeForThisRawWord = currWordLookupNode.getChildNode(token.rawWord);
                    //	If we can't continue this way...
                    if (nodeForThisRawWord == null)
                    {
                        //...then whatever our last good match was is the correct word.
                        if (lastGoodNode == null)
                        {
                            //	If there is no last good match, then take the raw words we've passed
                            //	and dump them all into raw word objects.
                            for (int i = lastProcessedTokenIdx + 1; i < tokenIdx; i++)
                            {
                                writePNode(new PNode(new RawWordObject(tokens[i].rawWord)));
                            }
                            lastProcessedTokenIdx = tokenIdx - 1;
                            //	Update everything so we continue after this point
                            //	Don't bother updating tokenIdx because we need to look at the word again.
                            //	Do update currWordLookupNode
                            currWordLookupNode = wordLookupRoot.getChildNode(token.rawWord);
                            //	If there's no node at all, we have to deal with it now
                            if (currWordLookupNode == null)
                            {
                                writePNode(new PNode(new RawWordObject(token.rawWord)));
                                lastGoodNodeTokenIdx  = tokenIdx;
                                isBuildingWord        = false;
                                currWordLookupNode    = wordLookupRoot;
                                lastProcessedTokenIdx = tokenIdx;
                                tokenIdx++;
                            }
                            else
                            {
                                isBuildingWord       = true;
                                lastGoodNodeTokenIdx = tokenIdx - 1;
                                tokenIdx++;
                            }
                            continue;
                        }

                        writePNode(new PNode(lastGoodNode.Value));
                        //	Reset everything so we're looking for a new word again.
                        lastGoodNode          = null;
                        currWordLookupNode    = wordLookupRoot;
                        isBuildingWord        = false;
                        lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                        //	Move the head back to the spot after the last token in the word
                        tokenIdx = lastGoodNodeTokenIdx + 1;
                        continue;
                    }

                    //	If adding this raw word makes a word, then record it as good
                    if (nodeForThisRawWord.Value != null)
                    {
                        lastGoodNode         = nodeForThisRawWord;
                        lastGoodNodeTokenIdx = tokenIdx;
                    }
                    currWordLookupNode = nodeForThisRawWord;
                    continue;
                }
                else
                {
                    switch (token.tokenType)
                    {
                    case LexerToken.TYPE.UNCLASSIFIED:
                    {
                        isBuildingWord = true;
                        //	Try to continue the current word matching.
                        Trie <RawWord, Word> .Node nodeForThisRawWord = currWordLookupNode.getChildNode(token.rawWord);
                        //	If we can't continue this way...
                        if (nodeForThisRawWord == null)
                        {
                            //...then whatever our last good match was is the correct word.
                            if (lastGoodNode == null)
                            {
                                //throw new RuntimeLexerSourceException("Unrecognized word or symbol.", token);
                                //	Dump everything into raw words.
                                tokenIdx++;                                     //	Include this word
                                for (int i = lastProcessedTokenIdx + 1; i < tokenIdx; i++)
                                {
                                    writePNode(new PNode(new RawWordObject(tokens[i].rawWord)));
                                }
                                //	Update everything so we continue after this point
                                lastGoodNodeTokenIdx  = tokenIdx - 1;
                                lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                                currWordLookupNode    = wordLookupRoot;
                                isBuildingWord        = false;
                                continue;
                            }
                            writePNode(new PNode(lastGoodNode.Value));
                            //	Reset everything so we're looking for a new word again.
                            lastGoodNode          = null;
                            currWordLookupNode    = wordLookupRoot;
                            isBuildingWord        = false;
                            lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                            //	Move the head back to the spot after the last token in the word
                            tokenIdx = lastGoodNodeTokenIdx + 1;
                            continue;
                        }

                        //	If adding this raw word makes a word, then record it as good
                        if (nodeForThisRawWord.Value != null)
                        {
                            lastGoodNode         = nodeForThisRawWord;
                            lastGoodNodeTokenIdx = tokenIdx;
                        }
                        currWordLookupNode = nodeForThisRawWord;
                        continue;
                    }
                    break;

                    case LexerToken.TYPE.STRING:
                    {
                        //	First clean up any word we may be building and write it to output.
                        if (isBuildingWord)
                        {
                            if (lastGoodNode == null || lastGoodNodeTokenIdx != tokenIdx - 1)
                            {
                                //throw new RuntimeLexerSourceException("Unrecognized word or symbol.", tokens[tokenIdx-1]);
                                //	Just take all the words up until now and dump them into raw words.
                                for (int i = lastProcessedTokenIdx + 1; i < tokenIdx; i++)
                                {
                                    writePNode(new PNode(new RawWordObject(tokens[i].rawWord)));
                                }
                                //	Update everything so we continue after this point
                                lastGoodNodeTokenIdx  = tokenIdx - 1;
                                lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                                currWordLookupNode    = wordLookupRoot;
                                isBuildingWord        = false;
                            }
                            else
                            {
                                writePNode(new PNode(lastGoodNode.Value));
                                //	Reset everything so we're looking for a new word again.
                                lastGoodNode          = null;
                                currWordLookupNode    = wordLookupRoot;
                                isBuildingWord        = false;
                                lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                            }
                        }

                        //	Now write the string literal object to output
                        writePNode(new PNode(new StringLiteralObject(token.rawWord.AsString)));
                        lastProcessedTokenIdx = tokenIdx;
                        //	Continue
                        tokenIdx++;
                        continue;
                    }
                    break;
                    }
                }
            }


            finalCheckAfterParsing();
            return(outputRoot);
        }
예제 #8
0
        public PNode parseTokensIntoPNodes(ProseRuntime runtime, ProseClient who, List<LexerToken> tokens)
        {
            //	Save the parameters
            this.who = who;
            this.runtime = runtime;
            sourceTokenList = tokens;

            setupForParsing();

            Trie<RawWord, Word>.Node currWordLookupNode, wordLookupRoot, lastGoodNode;
            wordLookupRoot = runtime.getWordLookupRoot();
            currWordLookupNode = wordLookupRoot;
            lastGoodNode = null;
            int lastGoodNodeTokenIdx = -1;
            //	Record whether or not we're in the process of building up a word.
            bool isBuildingWord = false;
            int lastProcessedTokenIdx = -1;

            while (tokenIdx < tokens.Count)
            {
                LexerToken token = tokens[tokenIdx];

                //
                //	Deal With Quadquotes.
                //
                if (token.rawWord == ProseLanguage.Raw.Quadquote) {
                    //	First clean up any word we may be building and write it to output.
                    if (isBuildingWord)
                    {
                        if (lastGoodNode == null)
                        {
            //							//	If we're inside a quadquote block then this is fine: even if we don't have
            //							//	a legitimate word we can still wrap rawwords.  Otherwise it's an error.
            //							if (insideQuadquoteExpression)
                            {
                                //	If there is no last good match, then take the raw words we've passed
                                //	and dump them all into raw word objects.
                                for (int i=lastProcessedTokenIdx + 1; i < tokenIdx; i++) {
                                    writePNode(new PNode(new RawWordObject(tokens[i].rawWord)));
                                }
                                //	Update everything so we continue after this point
                                lastGoodNodeTokenIdx = tokenIdx - 1;
                                lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                                currWordLookupNode = wordLookupRoot;
                                isBuildingWord = false;
                                //	Don't bother updating tokenIdx because we need to look at the word again.
                            }
            //							else {
            //								throw new RuntimeLexerSourceException("Unrecognized word or symbol.", tokens[tokenIdx-1]);
            //							}
                        }
                        else {
                            writePNode(new PNode(lastGoodNode.Value));
                            //	Reset everything so we're looking for a new word again.
                            lastGoodNode = null;
                            currWordLookupNode = wordLookupRoot;
                            isBuildingWord = false;
                            lastGoodNodeTokenIdx = tokenIdx;
                            lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                        }

                    }

                    //	Output a quadquote
                    writePNode(new PNode(runtime.Quadquote));
                    //	Toggle our quad-quote-state.
                    insideQuadquoteExpression = !insideQuadquoteExpression;
                    lastQuadquoteIdx = tokenIdx;
                    lastProcessedTokenIdx = tokenIdx;
                    //	Continue
                    tokenIdx++;
                    continue;
                }

                if (insideQuadquoteExpression)
                {
                    if (token.tokenType != LexerToken.TYPE.UNCLASSIFIED)
                        throw new RuntimeLexerFailure("Static Lexer Failed Token Classification.");

                    //
                    //	This code is essentially copied from the LexerToken.TYPE.UNCLASSIFIED block below.
                    //	The only major difference is that instead of throwing an exception we wrap unknown
                    //	text inside raw word objects.
                    //
                    isBuildingWord = true;
                    //	Try to continue the current word matching.
                    Trie<RawWord, Word>.Node nodeForThisRawWord = currWordLookupNode.getChildNode(token.rawWord);
                    //	If we can't continue this way...
                    if (nodeForThisRawWord == null)
                    {
                        //...then whatever our last good match was is the correct word.
                        if (lastGoodNode == null) {
                            //	If there is no last good match, then take the raw words we've passed
                            //	and dump them all into raw word objects.
                            for (int i=lastProcessedTokenIdx + 1; i < tokenIdx; i++) {
                                writePNode(new PNode(new RawWordObject(tokens[i].rawWord)));
                            }
                            lastProcessedTokenIdx = tokenIdx - 1;
                            //	Update everything so we continue after this point
                            //	Don't bother updating tokenIdx because we need to look at the word again.
                            //	Do update currWordLookupNode
                            currWordLookupNode = wordLookupRoot.getChildNode(token.rawWord);
                            //	If there's no node at all, we have to deal with it now
                            if (currWordLookupNode == null)
                            {
                                writePNode(new PNode(new RawWordObject(token.rawWord)));
                                lastGoodNodeTokenIdx = tokenIdx;
                                isBuildingWord = false;
                                currWordLookupNode = wordLookupRoot;
                                lastProcessedTokenIdx = tokenIdx;
                                tokenIdx++;
                            }
                            else {
                                isBuildingWord = true;
                                lastGoodNodeTokenIdx = tokenIdx - 1;
                                tokenIdx++;
                            }
                            continue;
                        }

                        writePNode(new PNode(lastGoodNode.Value));
                        //	Reset everything so we're looking for a new word again.
                        lastGoodNode = null;
                        currWordLookupNode = wordLookupRoot;
                        isBuildingWord = false;
                        lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                        //	Move the head back to the spot after the last token in the word
                        tokenIdx = lastGoodNodeTokenIdx + 1;
                        continue;
                    }

                    //	If adding this raw word makes a word, then record it as good
                    if (nodeForThisRawWord.Value != null)
                    {
                        lastGoodNode = nodeForThisRawWord;
                        lastGoodNodeTokenIdx = tokenIdx;
                    }
                    currWordLookupNode = nodeForThisRawWord;
                    continue;
                }
                else {
                    switch (token.tokenType)
                    {
                    case LexerToken.TYPE.UNCLASSIFIED:
                    {
                        isBuildingWord = true;
                        //	Try to continue the current word matching.
                        Trie<RawWord, Word>.Node nodeForThisRawWord = currWordLookupNode.getChildNode(token.rawWord);
                        //	If we can't continue this way...
                        if (nodeForThisRawWord == null)
                        {
                            //...then whatever our last good match was is the correct word.
                            if (lastGoodNode == null) {
                                //throw new RuntimeLexerSourceException("Unrecognized word or symbol.", token);
                                //	Dump everything into raw words.
                                tokenIdx++;	//	Include this word
                                for (int i=lastProcessedTokenIdx + 1; i < tokenIdx; i++) {
                                    writePNode(new PNode(new RawWordObject(tokens[i].rawWord)));
                                }
                                //	Update everything so we continue after this point
                                lastGoodNodeTokenIdx = tokenIdx - 1;
                                lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                                currWordLookupNode = wordLookupRoot;
                                isBuildingWord = false;
                                continue;
                            }
                            writePNode(new PNode(lastGoodNode.Value));
                            //	Reset everything so we're looking for a new word again.
                            lastGoodNode = null;
                            currWordLookupNode = wordLookupRoot;
                            isBuildingWord = false;
                            lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                            //	Move the head back to the spot after the last token in the word
                            tokenIdx = lastGoodNodeTokenIdx + 1;
                            continue;
                        }

                        //	If adding this raw word makes a word, then record it as good
                        if (nodeForThisRawWord.Value != null)
                        {
                            lastGoodNode = nodeForThisRawWord;
                            lastGoodNodeTokenIdx = tokenIdx;
                        }
                        currWordLookupNode = nodeForThisRawWord;
                        continue;
                    }
                    break;

                    case LexerToken.TYPE.STRING:
                    {
                        //	First clean up any word we may be building and write it to output.
                        if (isBuildingWord)
                        {
                            if (lastGoodNode == null || lastGoodNodeTokenIdx != tokenIdx - 1) {
                                //throw new RuntimeLexerSourceException("Unrecognized word or symbol.", tokens[tokenIdx-1]);
                                //	Just take all the words up until now and dump them into raw words.
                                for (int i=lastProcessedTokenIdx + 1; i < tokenIdx; i++) {
                                    writePNode(new PNode(new RawWordObject(tokens[i].rawWord)));
                                }
                                //	Update everything so we continue after this point
                                lastGoodNodeTokenIdx = tokenIdx - 1;
                                lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                                currWordLookupNode = wordLookupRoot;
                                isBuildingWord = false;
                            }
                            else {
                                writePNode(new PNode(lastGoodNode.Value));
                                //	Reset everything so we're looking for a new word again.
                                lastGoodNode = null;
                                currWordLookupNode = wordLookupRoot;
                                isBuildingWord = false;
                                lastProcessedTokenIdx = lastGoodNodeTokenIdx;
                            }
                        }

                        //	Now write the string literal object to output
                        writePNode(new PNode(new StringLiteralObject(token.rawWord.AsString)));
                        lastProcessedTokenIdx = tokenIdx;
                        //	Continue
                        tokenIdx++;
                        continue;
                    }
                    break;
                    }
                }
            }

            finalCheckAfterParsing();
            return outputRoot;
        }