Example #1
0
        public void TestStreamTokenizer()
        {
            String source = "123456789ABCDEF";
            Func <List <String> > input = () => source.ToCharArray()
                                          .Select(i => i.ToString(CultureInfo.InvariantCulture))
                                          .ToList();
            TokenizableDataStream <String> data = new TokenizableDataStream <String>(input);

            // Take base snapshot for later match
            data.PushSnapshot();
            // Match CurrentItem result
            Assert.AreEqual(data.CurrentItem, "1");
            // Match advance
            data.Advance(3);
            Assert.AreEqual(data.CurrentItem, "4");
            // Snapshot and advance and then pop and ensure result
            data.PushSnapshot();
            data.Advance();
            data.PopSnapshot();
            Assert.AreEqual(data.CurrentItem, "4");
            // Snapshot, advance, discard snapshot and ensure result
            data.PushSnapshot();
            data.Advance(3);
            data.DiscardSnapshot();
            Assert.AreEqual(data.CurrentItem, "7");
            // Pop snapshot and assert that we are once again at the start
            data.PopSnapshot();
            Assert.AreEqual(data.CurrentItem, "1");
            // Test AtEnd for non-eof
            Assert.IsFalse(data.AtEnd());
            // Advance too far and ensure that AtEnd returns
            data.PushSnapshot();
            data.Advance(23);
            Assert.IsTrue(data.AtEnd());
            // Assert that currentItem is null when out of range
            Assert.IsNull(data.CurrentItem);
            // Pop back to start and assert that LookAhead returns desired item
            data.PopSnapshot();
            Assert.AreEqual(data.LookAhead(4), "5");
            // Assert that LookAhead returns null once out of range
            Assert.IsNull(data.LookAhead(42));
        }
Example #2
0
        public void TestStreamTokenizer()
        {
            String source = "123456789ABCDEF";
            Func<List<String>> input = () => source.ToCharArray()
                .Select(i => i.ToString(CultureInfo.InvariantCulture))
                .ToList();
            TokenizableDataStream<String> data = new TokenizableDataStream<String>(input);

            // Take base snapshot for later match
            data.PushSnapshot();
            // Match CurrentItem result
            Assert.AreEqual(data.CurrentItem, "1");
            // Match advance
            data.Advance(3);
            Assert.AreEqual(data.CurrentItem, "4");
            // Snapshot and advance and then pop and ensure result
            data.PushSnapshot();
            data.Advance();
            data.PopSnapshot();
            Assert.AreEqual(data.CurrentItem, "4");
            // Snapshot, advance, discard snapshot and ensure result
            data.PushSnapshot();
            data.Advance(3);
            data.DiscardSnapshot();
            Assert.AreEqual(data.CurrentItem, "7");
            // Pop snapshot and assert that we are once again at the start
            data.PopSnapshot();
            Assert.AreEqual(data.CurrentItem, "1");
            // Test AtEnd for non-eof
            Assert.IsFalse(data.AtEnd());
            // Advance too far and ensure that AtEnd returns
            data.PushSnapshot();
            data.Advance(23);
            Assert.IsTrue(data.AtEnd());
            // Assert that currentItem is null when out of range
            Assert.IsNull(data.CurrentItem);
            // Pop back to start and assert that LookAhead returns desired item
            data.PopSnapshot();
            Assert.AreEqual(data.LookAhead(4), "5");
            // Assert that LookAhead returns null once out of range
            Assert.IsNull(data.LookAhead(42));
        }
Example #3
0
        protected override Token <string> Match(TokenizableDataStream <string> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);
            string         peek  = data.CurrentItem;
            string         word  = null;

loopStart:
            while (!data.AtEnd() && !string.IsNullOrWhiteSpace(peek) &&
                   (Delimiters.All(d => d.Keyword != peek)) &&
                   peek != "\"" && peek != "'")
            {
                word += peek;
                data.Advance();
                peek = data.CurrentItem;
            }

            //HACK: there are variable names that include the c++ scope operator '::' for some godforsaken reason
            if (peek == ":" && data.LookAhead(1) == ":")
            {
                word += peek;
                data.Advance();
                peek  = data.CurrentItem;
                word += peek;
                data.Advance();
                peek = data.CurrentItem;
                goto loopStart;
            }

            if (word != null)
            {
                streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                SourcePosition end = new SourcePosition(streamPos);
                return(new Token <string>(TokenType.Word, word, start, end));
            }
            return(null);
        }