Пример #1
0
        public void RemoveAllSingleNodes()
        {
            const string code              = @"class K { void m1() {} void m2() { int i; } }";
            var          cst               = CstGenerators.JavaUsingAntlr3.GenerateTreeFromCodeText(code);
            var          nodes             = cst.Descendants().ToList();
            var          originalNodeCount = nodes.Count;

            foreach (var node in nodes)
            {
                if (node.Parent == null)
                {
                    continue;
                }
                var single = node.DescendantsOfSingle().LastOrDefault();
                if (single != null && node.FirstChild != single)
                {
                    if (single.HasToken)
                    {
                        var newToken = new CstToken(node.Name, single.Token.Text, single.Token.RuleId);
                        node.Replace(new CstNode(newToken, single.Hiddens));
                    }
                    else
                    {
                        single.Remove();
                        node.FirstChild.Remove();
                        node.AddFirst(single);
                    }
                }
            }
            Console.WriteLine(originalNodeCount + ", " + cst.Descendants().Count());
            var actualCode = cst.Code;

            Assert.That(actualCode, Is.EqualTo(code));
        }
Пример #2
0
        protected override void ApplyMaps(CstToken token)
        {
            SimpleMapCollection c = SimpleMap.VBNetMaps;

            foreach (SimpleMap map in c)
            {
                if (map.IsRegExp)
                {
                    Regex regex = new Regex(map.Source);
                    token.Text = regex.Replace(token.Text, map.Target);
                }
                else
                {
                    token.Text = token.Text.Replace(map.Source, map.Target);
                }
            }
        }
Пример #3
0
		public void RemoveAllSingleNodes() {
			const string code = @"class K { void m1() {} void m2() { int i; } }";
			var cst = CstGenerators.JavaUsingAntlr3.GenerateTreeFromCodeText(code);
			var nodes = cst.Descendants().ToList();
			var originalNodeCount = nodes.Count;
			foreach (var node in nodes) {
				if (node.Parent == null) {
					continue;
				}
				var single = node.DescendantsOfSingle().LastOrDefault();
				if (single != null && node.FirstChild != single) {
					if (single.HasToken) {
						var newToken = new CstToken(node.Name, single.Token.Text, single.Token.RuleId);
						node.Replace(new CstNode(newToken, single.Hiddens));
					} else {
						single.Remove();
						node.FirstChild.Remove();
						node.AddFirst(single);
					}
				}
			}
			Console.WriteLine(originalNodeCount + ", " + cst.Descendants().Count());
			var actualCode = cst.Code;
			Assert.That(actualCode, Is.EqualTo(code));
		}
        public void SetLocationInformation(CstNode tree, string code) {
            var line = 1;
            var pos = 0;
            var index = 0;
            var startHiddenLocation = new CodeLocation(line, pos);
            foreach (var tokenNode in tree.AllTokenNodes()) {
                var token = tokenNode.Token;
                var tokenStr = token.Text;
                if (tokenStr == string.Empty) { // e.g. NEWLINE in Python
                    token.Range = new CodeRange(startHiddenLocation, startHiddenLocation);
                    continue;
                }
                var startHiddenIndex = index;
                var tokenChar = tokenStr[0];
                while (code[index] != tokenChar
                       || code.Substring(index, tokenStr.Length) != tokenStr) {
                    if (code[index] != '\n') {
                        pos++;
                    } else {
                        line++;
                        pos = 0;
                    }
                    index++;
                }
                var endHiddenIndex = index;
                var startTokenLocation = new CodeLocation(line, pos);
                var endIndex = index + tokenStr.Length;
                while (index < endIndex) {
                    if (code[index] != '\n') {
                        pos++;
                    } else {
                        line++;
                        pos = 0;
                    }
                    index++;
                }
                tokenNode.Hiddens.Add(
                        new CstToken(
                                Code2XmlConstants.HiddenElementName,
                                code.Substring(startHiddenIndex, endHiddenIndex - startHiddenIndex),
                                Code2XmlConstants.DefaultHiddenRuleId,
                                new CodeRange(startHiddenLocation, startTokenLocation)));
                startHiddenLocation = new CodeLocation(line, pos); // as endTokenLocation
                token.Range = new CodeRange(startTokenLocation, startHiddenLocation);
            }

            {
                var startHiddenIndex = index;
                var endIndex = code.Length;
                while (index < endIndex) {
                    if (code[index] != '\n') {
                        pos++;
                    } else {
                        line++;
                        pos = 0;
                    }
                    index++;
                }
                var endHiddenLocation = new CodeLocation(line, pos);
                var token = new CstToken(
                        Code2XmlConstants.EofTokenName, String.Empty, Code2XmlConstants.EofRuleId,
                        new CodeRange(endHiddenLocation, endHiddenLocation));
                var eofNode = new CstNode(token);
                eofNode.Hiddens.Add(
                        new CstToken(
                                Code2XmlConstants.HiddenElementName,
                                code.Substring(startHiddenIndex, index - startHiddenIndex),
                                Code2XmlConstants.DefaultHiddenRuleId,
                                new CodeRange(startHiddenLocation, endHiddenLocation)));
                tree.AddLast(eofNode);
            }
        }
Пример #5
0
        private void BuildTokens()
        {
            cstText = cstText.TrimStart();

            log.AddEntry("Building Token List...");

            Regex ex = new Regex(REGEXP_TOKENIZE, (RegexOptions.Multiline | RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture));

            int      lastIndex = 0, currentIndex = 0;
            string   data, rawtag;
            int      commentListIndex = 0, scriptListIndex = 0, aspListIndex = 0;
            CstToken token = null;

            while (lastIndex >= 0)
            {
                Match match = ex.Match(cstText, lastIndex);

                if (match.Success)
                {
                    log.AddEntry("--> Found token");

                    currentIndex = match.Index;

                    if (currentIndex > lastIndex)
                    {
                        token = new CstToken(CstTokenType.Literal, cstText.Substring(lastIndex, currentIndex - lastIndex));
                        template.Tokens.Add(token);
                    }

                    rawtag = match.Value;
                    switch (rawtag)
                    {
                    case CstParser.TMP_START:
                        token = new CstToken(CstTokenType.EscapedStartTag, "<%");
                        break;

                    case CstParser.TMP_END:
                        token = new CstToken(CstTokenType.EscapedEndTag, "%>");
                        break;

                    case CstParser.TMP_COMMENT:
                        data  = comments[commentListIndex].ToString();
                        token = new CstToken(CstTokenType.Comment, data);
                        commentListIndex++;
                        break;

                    case CstParser.TMP_SCRIPT:
                        data  = scriptBlocks[scriptListIndex].ToString();
                        token = new CstToken(CstTokenType.RunAtServerCode, data);
                        scriptListIndex++;
                        break;

                    case CstParser.TMP_ASP:
                        data = aspBlocks[aspListIndex].ToString();
                        if (data.StartsWith("="))
                        {
                            token = new CstToken(CstTokenType.ResponseWriteShortcutCode, data.Remove(0, 1));
                        }
                        else
                        {
                            token = new CstToken(CstTokenType.Code, data);
                        }
                        aspListIndex++;
                        break;
                    }
                    template.Tokens.Add(token);

                    lastIndex = currentIndex + rawtag.Length;
                }
                else
                {
                    data  = cstText.Substring(lastIndex);
                    token = new CstToken(CstTokenType.Literal, data);
                    template.Tokens.Add(token);

                    lastIndex = -1;
                }
            }
        }
Пример #6
0
 protected abstract void ApplyMaps(CstToken token);
        public void SetLocationInformation(CstNode tree, string code)
        {
            var line  = 1;
            var pos   = 0;
            var index = 0;
            var startHiddenLocation = new CodeLocation(line, pos);

            foreach (var tokenNode in tree.AllTokenNodes())
            {
                var token    = tokenNode.Token;
                var tokenStr = token.Text;
                if (tokenStr == string.Empty)   // e.g. NEWLINE in Python
                {
                    token.Range = new CodeRange(startHiddenLocation, startHiddenLocation);
                    continue;
                }
                var startHiddenIndex = index;
                var tokenChar        = tokenStr[0];
                while (code[index] != tokenChar ||
                       code.Substring(index, tokenStr.Length) != tokenStr)
                {
                    if (code[index] != '\n')
                    {
                        pos++;
                    }
                    else
                    {
                        line++;
                        pos = 0;
                    }
                    index++;
                }
                var endHiddenIndex     = index;
                var startTokenLocation = new CodeLocation(line, pos);
                var endIndex           = index + tokenStr.Length;
                while (index < endIndex)
                {
                    if (code[index] != '\n')
                    {
                        pos++;
                    }
                    else
                    {
                        line++;
                        pos = 0;
                    }
                    index++;
                }
                tokenNode.Hiddens.Add(
                    new CstToken(
                        Code2XmlConstants.HiddenElementName,
                        code.Substring(startHiddenIndex, endHiddenIndex - startHiddenIndex),
                        Code2XmlConstants.DefaultHiddenRuleId,
                        new CodeRange(startHiddenLocation, startTokenLocation)));
                startHiddenLocation = new CodeLocation(line, pos); // as endTokenLocation
                token.Range         = new CodeRange(startTokenLocation, startHiddenLocation);
            }

            {
                var startHiddenIndex = index;
                var endIndex         = code.Length;
                while (index < endIndex)
                {
                    if (code[index] != '\n')
                    {
                        pos++;
                    }
                    else
                    {
                        line++;
                        pos = 0;
                    }
                    index++;
                }
                var endHiddenLocation = new CodeLocation(line, pos);
                var token             = new CstToken(
                    Code2XmlConstants.EofTokenName, String.Empty, Code2XmlConstants.EofRuleId,
                    new CodeRange(endHiddenLocation, endHiddenLocation));
                var eofNode = new CstNode(token);
                eofNode.Hiddens.Add(
                    new CstToken(
                        Code2XmlConstants.HiddenElementName,
                        code.Substring(startHiddenIndex, index - startHiddenIndex),
                        Code2XmlConstants.DefaultHiddenRuleId,
                        new CodeRange(startHiddenLocation, endHiddenLocation)));
                tree.AddLast(eofNode);
            }
        }