public void SetBlock(string id, Block value) { if (blockmap.ContainsKey(id)) { blockmap[id] = value; } else { blockmap.Add(id, value); } }
public Block ParseLine(string id, Line line, Block b, ref int _cmt, int start, int end, bool ispart) { tokentype = TokenType.TXT; tokens = new List<Token>(); List<Tuple<int, int, bool>> cmstrulrs = new List<Tuple<int, int, bool>>(); line.Block.isLineHeadCmt = _cmt; lex.Src = line.Text; if (ispart) { lex.SetRange(start, end); } //while (tokentype != TokenType.EOS) { while (lex.advance(b, line.Block)) { tokentype = lex.token; //if (lex.advance(b, line.Block)) { // tokentype = lex.token; //} //else { // tokentype = TokenType.EOS; //} switch (tokentype) { case TokenType.EndLine: case TokenType.Line: case TokenType.Enclose: case TokenType.Keyword: { tokens.Add(new Token { id = id, ad = lex.OffsetLenAttr.t1, len = lex.OffsetLenAttr.t2, attr = lex.OffsetLenAttr.t3.attr }); } break; case TokenType.MultiLineStart: { int off = lex.Offset; //int len = line.Length - lex.OffsetLenAttr.t1; int len = end - lex.OffsetLenAttr.t1; lex.isNextLine = true; cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = lex.isNextLine }); var parid = ((MultiLineRule)(lex.OffsetLenAttr.t3)).id; tokens.Add(new Token { id = parid, type = TokenType.MultiLine, mtype= MultiLineType.Start, ad = lex.OffsetLenAttr.t1, len = len, attr = lex.OffsetLenAttr.t3.attr }); } break; case TokenType.MultiLineAllLine: { int off = lex.Offset; //int len = line.Length - lex.OffsetLenAttr.t1; int len = end - lex.OffsetLenAttr.t1; lex.isNextLine = true; cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = lex.isNextLine }); var parid = ((MultiLineRule)(lex.OffsetLenAttr.t3)).id; if (line.Length == 0) { tokens.Add(new Token { id = parid, type = TokenType.MultiLine, mtype = MultiLineType.All, ad = 0, len = 0, attr = lex.OffsetLenAttr.t3.attr }); } else { tokens.Add(new Token { id = parid, type = TokenType.MultiLine, mtype = MultiLineType.All, ad = lex.OffsetLenAttr.t1, len = len, attr = lex.OffsetLenAttr.t3.attr }); } } break; case TokenType.MultiLineEnd: { //int len = line.Length - lex.OffsetLenAttr.t1; int len = end - lex.OffsetLenAttr.t1; bool isnext = false;// lex.isNextLine; lex.isNextLine = false; if (cmstrulrs.Count > 0) { cmstrulrs[cmstrulrs.Count - 1].t3 = isnext; } else { int off = lex.Offset; cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = isnext }); } if (tokens.Count > 0 && (tokens[tokens.Count - 1].mtype== MultiLineType.Start || tokens[tokens.Count - 1].mtype== MultiLineType.All)) { int off = tokens[tokens.Count - 1].ad; //tokens[tokens.Count - 1].len = off + lex.OffsetLenAttr.t2; tokens[tokens.Count - 1].mtype = MultiLineType.Line; tokens[tokens.Count - 1].len = lex.OffsetLenAttr.t2 - off; } else if (line.Block.isLineHeadCmt != 0) { var parid = ((MultiLineRule)(lex.OffsetLenAttr.t3)).id; tokens.Add(new Token { id = parid, type = TokenType.MultiLine, mtype = MultiLineType.End, ad = lex.OffsetLenAttr.t1, len = lex.OffsetLenAttr.t2, attr = lex.OffsetLenAttr.t3.attr }); } } break; default: break; } if (line.Length == 0) { break; } } if (cmstrulrs.Count == 0) { line.Block.commentTransition = 2; } else { bool next = cmstrulrs[cmstrulrs.Count - 1].t3; if (next) { line.Block.commentTransition = 3; } else { line.Block.commentTransition = 0; } } _cmt = (line.Block.commentTransition >> _cmt) & 1; if (tokens.Count > 0) { var lastrule = tokens[tokens.Count - 1]; //if (lastrule.ad + lastrule.len < line.Length) { // tokens.Add(new Token { id = id, ad = lastrule.ad + lastrule.len, len = line.Length - (lastrule.ad + lastrule.len), attr = defaultAttr }); //} if (lastrule.ad + lastrule.len < end) { tokens.Add(new Token { id = id, ad = lastrule.ad + lastrule.len, len = end - (lastrule.ad + lastrule.len), attr = defaultAttr }); } List<Token> defaultRules = new List<Token>(); int index = 0; for (int i = 0; i < tokens.Count; i++) { if (tokens[i].ad - index > 0) { defaultRules.Add(new Token { id = id, ad = index, len = tokens[i].ad - index, attr = defaultAttr }); } index = tokens[i].ad + tokens[i].len; } if (defaultRules.Count > 0) { tokens.AddRange(defaultRules); tokens.Sort((x, y) => { return x.ad < y.ad ? -1 : 1; }); } } else { if (ispart) { tokens.Add(new Token { id = id, ad = start, len = end - start, attr = defaultAttr }); } else { //tokens.Add(new Token { id = id, ad = 0, len = line.Length, attr = defaultAttr }); tokens.Add(new Token { id = id, ad = 0, len = end, attr = defaultAttr }); } } //} if (ispart) { foreach (var token in line.Tokens) { if (token.ad == start && (token.ad + token.len) == end) { int index = line.Tokens.IndexOf(token); line.Tokens.Remove(token); tokens[0].type = token.type; line.Tokens.InsertRange(index, tokens); break; } } } else { line.Tokens = tokens; } return line.Block; }
public Block Parse(Line line, Block b, int _cmt, int _sccmt) { tokentype = TokenType.TXT; tokens = new List<Token>(); List<Tuple<int, int, bool>> cmstrulrs = new List<Tuple<int, int, bool>>(); line.Block.isLineHeadCmt = _cmt; bool? isscnext = null; line.Block.isLineHeadPart = _sccmt; if (line.Block.isLineHeadPart == 0) { line.Block.PartID = Document.DEFAULT_ID; setd(b.PartID); } else { line.Block.PartID = b.PartID; setd(b.PartID); } lex.Src = line.Text; while (tokentype != TokenType.EOS) { if (lex.advance(b, line.Block)) { tokentype = lex.token; } else { tokentype = TokenType.EOS; } switch (tokentype) { case TokenType.EndLine: case TokenType.Line: case TokenType.Enclose: case TokenType.Keyword: { tokens.Add(new Token { ad = lex.OffsetLenAttr.t1, len = lex.OffsetLenAttr.t2, attr = lex.OffsetLenAttr.t3 }); } break; case TokenType.MultiLineStart: { int off = lex.Offset; int len = line.Length - lex.OffsetLenAttr.t1; lex.isNextLine = true; cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = lex.isNextLine }); tokens.Add(new Token { ad = lex.OffsetLenAttr.t1, len = len, attr = lex.OffsetLenAttr.t3 }); } break; case TokenType.MultiLineEnd: { int len = line.Length - lex.OffsetLenAttr.t1; bool isnext = false;// lex.isNextLine; lex.isNextLine = false; if (cmstrulrs.Count > 0) { cmstrulrs[cmstrulrs.Count - 1].t3 = isnext; } else { int off = lex.Offset; cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = isnext }); } if (tokens.Count > 0) { int off = tokens[tokens.Count - 1].ad; //tokens[tokens.Count - 1].len = off + lex.OffsetLenAttr.t2; tokens[tokens.Count - 1].len = lex.OffsetLenAttr.t2 - off; } else if (line.Block.isLineHeadCmt!=0) { tokens.Add(new Token { ad = lex.OffsetLenAttr.t1, len = lex.OffsetLenAttr.t2, attr = lex.OffsetLenAttr.t3 }); } } break; case TokenType.PartitionStart: isscnext = lex.scisNextLine; if (line.Block.PartID != Document.DEFAULT_ID) { setd(line.Block.PartID); } break; case TokenType.Partition: isscnext = lex.scisNextLine; setd(line.Block.PartID); break; case TokenType.PartitionEnd: isscnext = lex.scisNextLine; setd(Document.DEFAULT_ID); break; default: break; } } if (cmstrulrs.Count == 0) { line.Block.commentTransition = 2; } else { bool next = cmstrulrs[cmstrulrs.Count - 1].t3; if (next) { line.Block.commentTransition = 3; } else { line.Block.commentTransition = 0; } } cmt = (line.Block.commentTransition >> _cmt) & 1; if (isscnext == null) { line.Block.partTransition = 2; } else { if ((bool)isscnext) { line.Block.partTransition = 3; } else { line.Block.partTransition = 0; } } sccmt = (line.Block.partTransition >> _sccmt) & 1; if (tokens.Count > 0) { var lastrule = tokens[tokens.Count - 1]; if (lastrule.ad + lastrule.len < line.Length) { tokens.Add(new Token { ad = lastrule.ad + lastrule.len, len = line.Length - (lastrule.ad + lastrule.len), attr = defaultAttr }); } List<Token> defaultRules = new List<Token>(); int index = 0; for (int i = 0; i < tokens.Count; i++) { if (tokens[i].ad - index > 0) { defaultRules.Add(new Token { ad = index, len = tokens[i].ad - index, attr = defaultAttr }); } index = tokens[i].ad + tokens[i].len; } if (defaultRules.Count > 0) { tokens.AddRange(defaultRules); tokens.Sort((x, y) => { return x.ad < y.ad ? -1 : 1; }); } } else { tokens.Add(new Token { ad = 0, len = line.Length, attr = defaultAttr }); } line.Tokens = tokens; return line.Block; }
public Block ParseLine(string id, Line line, Block b, ref int _cmt) { return this.ParseLine(id, line, b, ref _cmt, 0, line.Length, false); }
public bool Parse(List<Line> text, int ad, int s, int e) { var id = getToken(text[s], ad); Line.ID = id; SetPartition(id, false); int i; Block block = text[s].Block; int cmt = text[s].Block.isLineHeadCmt; if (s > 0) { block = text[s - 1].Block; //TODO } if (id != Document.DEFAULT_ID) { var tmpid = id; var idlist = new List<string>(); while (true) { idlist.Add(tmpid); var parent = getPartition(tmpid).Parent; if (parent == null) { idlist.Add(Document.DEFAULT_ID); break; } else if (parent.ID == Document.DEFAULT_ID) { idlist.Add(Document.DEFAULT_ID); break; } tmpid = parent.ID; } //var pid = parser.getPartition(id).Parent.ID; for (int j = s; j <= e; ++j) { if (text[j].GetBlockLength() < idlist.Count) { foreach (var item in idlist) { var pb = text[j].GetBlock(item); if (pb == null) { Block nb = new Block(); nb.isLineHeadCmt = 1; nb.commentTransition = 3; if (j > 0) { nb.mRule = text[j - 1].GetBlock(item).mRule; } text[j].SetBlock(item, nb); } } } } } // まずは変更範囲を再解析 for (i = s; i <= e; ++i) { block = ParseLine(id, text[i], block, ref cmt); //cmt = parser.cmt; } //if (text_[i - 1].Block.mRule != null && id != text_[i - 1].Block.mRule.id) { // return false; //} // コメントアウト状態に変化がなかったらここでお終い。 //if (i == tln() || text_[i].Block.isLineHeadCmt == cmt) //if (i == tln() || (text_[i].Block.isLineHeadCmt == cmt && text_[i].Block.mRule == block.mRule)) { //if (i == tln() || (text_[i].Block.isLineHeadCmt == cmt && text_[i].Block.mRule == block.mRule) // || (text_[i].Block.mRule != null && id != text_[i].Block.mRule.id)) { int tln = text.Count(); if (id != Document.DEFAULT_ID) { tln = lines(text, ad, s, e); //ll++; //int ii = 0; } if (i == tln || (text[i].Block.isLineHeadCmt == cmt && text[i].Block.mRule == block.mRule)) { //if (i == tln() || (text_[i].Block.isLineHeadCmt == cmt && text_[i].Block.mRule == block.mRule)) { //var lists = lines(s, tln()==ll?e:ll); var lists = lines(text, s, tln); foreach (var list in lists) { ParsePart(text, list); } return false; } int pcmt = 0; Rule prule = null; // 例えば、/* が入力された場合などは、下の方の行まで // コメントアウト状態の変化を伝達する必要がある。 do { Line line = text[i++]; pcmt = line.Block.isLineHeadCmt; prule = line.Block.mRule; if (i == tln) { block = ParseLine(id, line, block, ref cmt, 0, las, true); } else { block = ParseLine(id, line, block, ref cmt); } //cmt = parser.cmt; if (pcmt == cmt) { if (prule != block.mRule) { pcmt--; } } //} while (i < tln() && pcmt != cmt); } while (i < tln && pcmt != cmt); int ss = s; int se = i; return true; }
private void lexSymbol(Block curblock) { StringBuilder buf = new StringBuilder(); int offset = reader.offset() - 1; while (true) { int c = reader.read(); if (c == -1) { return; } buf.Append((char)c); string s = buf.ToString(); if (ruleDic.ContainsKey(s)) { var rule = ruleDic[s]; if (rule.Detected(s, reader)) { tok = rule.token; int len = rule.getLen(s, reader); OffsetLenAttr = new Tuple<int, int, Attribute>(offset, len, rule.attr); reader.setoffset(offset + len); if (rule is MultiLineRule) { if (curblock != null) { curblock.mRule = rule as MultiLineRule; //isNextLine = true; } } break; } } else if (multiRuleEndDic.ContainsKey(s)) { var rule = multiRuleEndDic[s]; if (rule.Detected(s, reader)) { tok = rule.token; int len = rule.getLen(s, reader); OffsetLenAttr = new Tuple<int, int, Attribute>(offset, len, rule.attr); reader.setoffset(offset + len); if (curblock != null) { curblock.mRule = rule as MultiLineRule; } break; } } } }
public bool advance(Block preblock, Block curblock) { tok = TokenType.TXT; if (Src.Length == 0 && curblock.isLineHeadCmt == 1) { curblock.mRule = preblock.mRule; isNextLine = true; } if (Src.Length == 0 && curblock.isLineHeadPart == 1) { curblock.PartID = preblock.PartID; scisNextLine = true; } int c = reader.read(); if (c == -1) { tok = TokenType.EOS; return false; } switch (c) { case ' ': case 0x3000: case '\t': break; default: //TODO test if (curblock.isLineHeadCmt == 0) { if (curblock.isLineHeadPart == 0) { if (paruleStartKeys.Contains((char)c)) { char fc = (char)c; foreach (var item in partRuleDic) { if (item.Key[0] == fc) { var len = item.Value.start.Length; if (Offset - 1 + len <= Src.Length) { var text = Src.Substring(Offset - 1, len); if (text.ToString() == item.Key) { tok = TokenType.PartitionStart; if (curblock != null) { curblock.PartID = item.Value.id; scisNextLine = true; } reader.setoffset(Offset + text.Length); return true; } } } } } }else { if (Offset - 1 == 0) { while (c != -1) { if (paruleStartKeys.Contains((char)c)) { StringBuilder buf = new StringBuilder(); while (c != -1) { buf.Append((char)c); if (partRuleEndDic.ContainsKey(buf.ToString())) { if (preblock.PartID == partRuleEndDic[buf.ToString()].id) { var Eenelem = partRuleEndDic[buf.ToString()]; tok = TokenType.PartitionEnd; curblock.PartID = Eenelem.id; scisNextLine = false; reader.setoffset(Offset + buf.ToString().Length); return true; } } c = reader.read(); } } c = reader.read(); } //Finish: if (c == -1) { tok = TokenType.Partition; curblock.PartID = preblock.PartID; scisNextLine = true; } reader.setoffset(0); c = reader.read(); } else if (paruleStartKeys.Contains((char)c)) { char fc = (char)c; foreach (var item in partRuleDic) { if (item.Key[0] == fc) { var len = item.Value.start.Length; if (Offset - 1 + len <= Src.Length) { var text = Src.Substring(Offset - 1, len); if (text.ToString() == item.Key) { tok = TokenType.PartitionStart; if (curblock != null) { curblock.PartID = item.Key; scisNextLine = true; } reader.setoffset(Offset + text.Length); //break; return true; } } } } } } } if (curblock.isLineHeadCmt == 0) { //0: 行頭がブロックコメントの内部ではない if (Char.IsDigit((char)c)) { reader.unread(); lexDigit(); break; } else if (Util.isIdentifierPart((char)c)) { reader.unread(); lexKeyWord(); break; } else { reader.unread(); lexSymbol(curblock); break; } } else { //1: 行頭がブロックコメントの内部 if (Offset - 1 == 0) { reader.unread(); StringBuilder buf = new StringBuilder(); while (true) { c = reader.read(); if (c == -1) { break; } buf.Append((char)c); string s = buf.ToString(); string end = preblock.mRule.end; if (s.EndsWith(end)) { if (multiRuleEndDic.ContainsKey(end)) { var rule = multiRuleEndDic[end]; if (rule.Detected(end, reader)) { curblock.mRule = rule; //isNextLine = false; tok = rule.token; int len = rule.getLen(end, reader); reader.setoffset(len); OffsetLenAttr = new Tuple<int, int, Attribute>(0, len, rule.attr); //break; return true; } } } } if (c == -1 && preblock.mRule != null && multiRuleDic.ContainsKey(preblock.mRule.start)) { var enelem = multiRuleDic[preblock.mRule.start]; curblock.mRule = preblock.mRule; tok = TokenType.MultiLineStart; OffsetLenAttr = new Tuple<int, int, Attribute>(0, Src.Length, enelem.attr); reader.setoffset(Src.Length); } break; } else { if (Char.IsDigit((char)c)) { reader.unread(); lexDigit(); } else if (Util.isIdentifierPart((char)c)) { reader.unread(); lexKeyWord(); } else { reader.unread(); lexSymbol(curblock); } } } break; } return true; }
public bool advance(Block preblock, Block curblock) { tok = TokenType.TXT; if (Src.Length == 0 && curblock.isLineHeadCmt == 1) { curblock.mRule = preblock.mRule; tok = TokenType.MultiLineAllLine; isNextLine = true; return true; } int c = reader.read(); if (c == -1) { tok = TokenType.EOS; return false; } switch (c) { case ' ': case 0x3000: case '\t': break; default: if (curblock.isLineHeadCmt == 0) { //0: 行頭がブロックコメントの内部ではない if (Char.IsDigit((char)c)) { reader.unread(); lexDigit(); break; } else if (Util.isIdentifierPart((char)c)) { reader.unread(); lexKeyWord(); break; } else { reader.unread(); lexSymbol(curblock); break; } } else { //1: 行頭がブロックコメントの内部 if (Offset - 1 == 0) { int index; if ((index = Src.IndexOf(preblock.mRule.end, 0)) >= 0) { string end = preblock.mRule.end; var rule = multiRuleEndDic[end]; if (rule.Detected(end, reader)) { curblock.mRule = rule; tok = TokenType.MultiLineEnd; //int len = rule.getLen(end, reader); int len = index + end.Length; reader.setoffset(len); OffsetLenAttr = new Tuple<int, int, Rule>(0, len, rule); return true; } } else if(multiRuleDic.ContainsKey(preblock.mRule.start)){ var rule = multiRuleDic[preblock.mRule.start]; curblock.mRule = preblock.mRule; tok = TokenType.MultiLineAllLine; OffsetLenAttr = new Tuple<int, int, Rule>(0, Src.Length, rule); reader.setoffset(Src.Length); return true; } //{ // reader.unread(); // StringBuilder buf = new StringBuilder(); // while (true) { // c = reader.read(); // if (c == -1) { // break; // } // buf.Append((char)c); // string s = buf.ToString(); // string end = preblock.mRule.end; // if (s.EndsWith(end)) { // if (multiRuleEndDic.ContainsKey(end)) { // var rule = multiRuleEndDic[end]; // if (rule.Detected(end, reader)) { // curblock.mRule = rule; // //isNextLine = false; // //tok = rule.token; // tok = TokenType.MultiLineEnd; // int len = rule.getLen(end, reader); // reader.setoffset(len); // OffsetLenAttr = new Tuple<int, int, Rule>(0, len, rule); // //break; // return true; // } // } // } // } // if (c == -1 && preblock.mRule != null && multiRuleDic.ContainsKey(preblock.mRule.start)) { // var enelem = multiRuleDic[preblock.mRule.start]; // curblock.mRule = preblock.mRule; // //isNextLine = true; // //tok = TokenType.MultiLineStart; // tok = TokenType.MultiLineAllLine; // OffsetLenAttr = new Tuple<int, int, Rule>(0, Src.Length, enelem); // reader.setoffset(Src.Length); // } // break; //} } else { if (Char.IsDigit((char)c)) { reader.unread(); lexDigit(); } else if (Util.isIdentifierPart((char)c)) { reader.unread(); lexKeyWord(); } else { reader.unread(); lexSymbol(curblock); } } } break; } return true; }