public void SetBlock(string id, Block value)
{
if (blockmap.ContainsKey(id)) {
blockmap[id] = value;
}
else {
blockmap.Add(id, value);
}
}
public Block ParseLine(string id, Line line, Block b, ref int _cmt, int start, int end, bool ispart)
{
tokentype = TokenType.TXT;
tokens = new List<Token>();
List<Tuple<int, int, bool>> cmstrulrs = new List<Tuple<int, int, bool>>();
line.Block.isLineHeadCmt = _cmt;
lex.Src = line.Text;
if (ispart) {
lex.SetRange(start, end);
}
//while (tokentype != TokenType.EOS) {
while (lex.advance(b, line.Block)) {
tokentype = lex.token;
//if (lex.advance(b, line.Block)) {
// tokentype = lex.token;
//}
//else {
// tokentype = TokenType.EOS;
//}
switch (tokentype) {
case TokenType.EndLine:
case TokenType.Line:
case TokenType.Enclose:
case TokenType.Keyword: {
tokens.Add(new Token { id = id, ad = lex.OffsetLenAttr.t1, len = lex.OffsetLenAttr.t2, attr = lex.OffsetLenAttr.t3.attr });
}
break;
case TokenType.MultiLineStart: {
int off = lex.Offset;
//int len = line.Length - lex.OffsetLenAttr.t1;
int len = end - lex.OffsetLenAttr.t1;
lex.isNextLine = true;
cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = lex.isNextLine });
var parid = ((MultiLineRule)(lex.OffsetLenAttr.t3)).id;
tokens.Add(new Token { id = parid, type = TokenType.MultiLine, mtype= MultiLineType.Start, ad = lex.OffsetLenAttr.t1, len = len, attr = lex.OffsetLenAttr.t3.attr });
}
break;
case TokenType.MultiLineAllLine: {
int off = lex.Offset;
//int len = line.Length - lex.OffsetLenAttr.t1;
int len = end - lex.OffsetLenAttr.t1;
lex.isNextLine = true;
cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = lex.isNextLine });
var parid = ((MultiLineRule)(lex.OffsetLenAttr.t3)).id;
if (line.Length == 0) {
tokens.Add(new Token { id = parid, type = TokenType.MultiLine, mtype = MultiLineType.All, ad = 0, len = 0, attr = lex.OffsetLenAttr.t3.attr });
}
else {
tokens.Add(new Token { id = parid, type = TokenType.MultiLine, mtype = MultiLineType.All, ad = lex.OffsetLenAttr.t1, len = len, attr = lex.OffsetLenAttr.t3.attr });
}
}
break;
case TokenType.MultiLineEnd: {
//int len = line.Length - lex.OffsetLenAttr.t1;
int len = end - lex.OffsetLenAttr.t1;
bool isnext = false;// lex.isNextLine;
lex.isNextLine = false;
if (cmstrulrs.Count > 0) {
cmstrulrs[cmstrulrs.Count - 1].t3 = isnext;
}
else {
int off = lex.Offset;
cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = isnext });
}
if (tokens.Count > 0 && (tokens[tokens.Count - 1].mtype== MultiLineType.Start || tokens[tokens.Count - 1].mtype== MultiLineType.All)) {
int off = tokens[tokens.Count - 1].ad;
//tokens[tokens.Count - 1].len = off + lex.OffsetLenAttr.t2;
tokens[tokens.Count - 1].mtype = MultiLineType.Line;
tokens[tokens.Count - 1].len = lex.OffsetLenAttr.t2 - off;
}
else if (line.Block.isLineHeadCmt != 0) {
var parid = ((MultiLineRule)(lex.OffsetLenAttr.t3)).id;
tokens.Add(new Token { id = parid, type = TokenType.MultiLine, mtype = MultiLineType.End, ad = lex.OffsetLenAttr.t1, len = lex.OffsetLenAttr.t2, attr = lex.OffsetLenAttr.t3.attr });
}
}
break;
default:
break;
}
if (line.Length == 0) {
break;
}
}
if (cmstrulrs.Count == 0) {
line.Block.commentTransition = 2;
}
else {
bool next = cmstrulrs[cmstrulrs.Count - 1].t3;
if (next) {
line.Block.commentTransition = 3;
}
else {
line.Block.commentTransition = 0;
}
}
_cmt = (line.Block.commentTransition >> _cmt) & 1;
if (tokens.Count > 0) {
var lastrule = tokens[tokens.Count - 1];
//if (lastrule.ad + lastrule.len < line.Length) {
// tokens.Add(new Token { id = id, ad = lastrule.ad + lastrule.len, len = line.Length - (lastrule.ad + lastrule.len), attr = defaultAttr });
//}
if (lastrule.ad + lastrule.len < end) {
tokens.Add(new Token { id = id, ad = lastrule.ad + lastrule.len, len = end - (lastrule.ad + lastrule.len), attr = defaultAttr });
}
List<Token> defaultRules = new List<Token>();
int index = 0;
for (int i = 0; i < tokens.Count; i++) {
if (tokens[i].ad - index > 0) {
defaultRules.Add(new Token { id = id, ad = index, len = tokens[i].ad - index, attr = defaultAttr });
}
index = tokens[i].ad + tokens[i].len;
}
if (defaultRules.Count > 0) {
tokens.AddRange(defaultRules);
tokens.Sort((x, y) => {
return x.ad < y.ad ? -1 : 1;
});
}
}
else {
if (ispart) {
tokens.Add(new Token { id = id, ad = start, len = end - start, attr = defaultAttr });
}
else {
//tokens.Add(new Token { id = id, ad = 0, len = line.Length, attr = defaultAttr });
tokens.Add(new Token { id = id, ad = 0, len = end, attr = defaultAttr });
}
}
//}
if (ispart) {
foreach (var token in line.Tokens) {
if (token.ad == start && (token.ad + token.len) == end) {
int index = line.Tokens.IndexOf(token);
line.Tokens.Remove(token);
tokens[0].type = token.type;
line.Tokens.InsertRange(index, tokens);
break;
}
}
} else {
line.Tokens = tokens;
}
return line.Block;
}
public Block Parse(Line line, Block b, int _cmt, int _sccmt)
{
tokentype = TokenType.TXT;
tokens = new List<Token>();
List<Tuple<int, int, bool>> cmstrulrs = new List<Tuple<int, int, bool>>();
line.Block.isLineHeadCmt = _cmt;
bool? isscnext = null;
line.Block.isLineHeadPart = _sccmt;
if (line.Block.isLineHeadPart == 0) {
line.Block.PartID = Document.DEFAULT_ID;
setd(b.PartID);
}
else {
line.Block.PartID = b.PartID;
setd(b.PartID);
}
lex.Src = line.Text;
while (tokentype != TokenType.EOS) {
if (lex.advance(b, line.Block)) {
tokentype = lex.token;
}
else {
tokentype = TokenType.EOS;
}
switch (tokentype) {
case TokenType.EndLine:
case TokenType.Line:
case TokenType.Enclose:
case TokenType.Keyword: {
tokens.Add(new Token { ad = lex.OffsetLenAttr.t1, len = lex.OffsetLenAttr.t2, attr = lex.OffsetLenAttr.t3 });
}
break;
case TokenType.MultiLineStart: {
int off = lex.Offset;
int len = line.Length - lex.OffsetLenAttr.t1;
lex.isNextLine = true;
cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = lex.isNextLine });
tokens.Add(new Token { ad = lex.OffsetLenAttr.t1, len = len, attr = lex.OffsetLenAttr.t3 });
}
break;
case TokenType.MultiLineEnd: {
int len = line.Length - lex.OffsetLenAttr.t1;
bool isnext = false;// lex.isNextLine;
lex.isNextLine = false;
if (cmstrulrs.Count > 0) {
cmstrulrs[cmstrulrs.Count - 1].t3 = isnext;
} else {
int off = lex.Offset;
cmstrulrs.Add(new Tuple<int, int, bool> { t1 = off, t2 = len, t3 = isnext });
}
if (tokens.Count > 0) {
int off = tokens[tokens.Count - 1].ad;
//tokens[tokens.Count - 1].len = off + lex.OffsetLenAttr.t2;
tokens[tokens.Count - 1].len = lex.OffsetLenAttr.t2 - off;
} else if (line.Block.isLineHeadCmt!=0) {
tokens.Add(new Token { ad = lex.OffsetLenAttr.t1, len = lex.OffsetLenAttr.t2, attr = lex.OffsetLenAttr.t3 });
}
}
break;
case TokenType.PartitionStart:
isscnext = lex.scisNextLine;
if (line.Block.PartID != Document.DEFAULT_ID) {
setd(line.Block.PartID);
}
break;
case TokenType.Partition:
isscnext = lex.scisNextLine;
setd(line.Block.PartID);
break;
case TokenType.PartitionEnd:
isscnext = lex.scisNextLine;
setd(Document.DEFAULT_ID);
break;
default:
break;
}
}
if (cmstrulrs.Count == 0) {
line.Block.commentTransition = 2;
}
else {
bool next = cmstrulrs[cmstrulrs.Count - 1].t3;
if (next) {
line.Block.commentTransition = 3;
}
else {
line.Block.commentTransition = 0;
}
}
cmt = (line.Block.commentTransition >> _cmt) & 1;
if (isscnext == null) {
line.Block.partTransition = 2;
}
else {
if ((bool)isscnext) {
line.Block.partTransition = 3;
}
else {
line.Block.partTransition = 0;
}
}
sccmt = (line.Block.partTransition >> _sccmt) & 1;
if (tokens.Count > 0) {
var lastrule = tokens[tokens.Count - 1];
if (lastrule.ad + lastrule.len < line.Length) {
tokens.Add(new Token { ad = lastrule.ad + lastrule.len, len = line.Length - (lastrule.ad + lastrule.len), attr = defaultAttr });
}
List<Token> defaultRules = new List<Token>();
int index = 0;
for (int i = 0; i < tokens.Count; i++) {
if (tokens[i].ad - index > 0) {
defaultRules.Add(new Token { ad = index, len = tokens[i].ad - index, attr = defaultAttr });
}
index = tokens[i].ad + tokens[i].len;
}
if (defaultRules.Count > 0) {
tokens.AddRange(defaultRules);
tokens.Sort((x, y) => {
return x.ad < y.ad ? -1 : 1;
});
}
}
else {
tokens.Add(new Token { ad = 0, len = line.Length, attr = defaultAttr });
}
line.Tokens = tokens;
return line.Block;
}
public Block ParseLine(string id, Line line, Block b, ref int _cmt)
{
return this.ParseLine(id, line, b, ref _cmt, 0, line.Length, false);
}
public bool Parse(List<Line> text, int ad, int s, int e)
{
var id = getToken(text[s], ad);
Line.ID = id;
SetPartition(id, false);
int i;
Block block = text[s].Block;
int cmt = text[s].Block.isLineHeadCmt;
if (s > 0) {
block = text[s - 1].Block; //TODO
}
if (id != Document.DEFAULT_ID) {
var tmpid = id;
var idlist = new List<string>();
while (true) {
idlist.Add(tmpid);
var parent = getPartition(tmpid).Parent;
if (parent == null) {
idlist.Add(Document.DEFAULT_ID);
break;
}
else if (parent.ID == Document.DEFAULT_ID) {
idlist.Add(Document.DEFAULT_ID);
break;
}
tmpid = parent.ID;
}
//var pid = parser.getPartition(id).Parent.ID;
for (int j = s; j <= e; ++j) {
if (text[j].GetBlockLength() < idlist.Count) {
foreach (var item in idlist) {
var pb = text[j].GetBlock(item);
if (pb == null) {
Block nb = new Block();
nb.isLineHeadCmt = 1;
nb.commentTransition = 3;
if (j > 0) {
nb.mRule = text[j - 1].GetBlock(item).mRule;
}
text[j].SetBlock(item, nb);
}
}
}
}
}
// まずは変更範囲を再解析
for (i = s; i <= e; ++i) {
block = ParseLine(id, text[i], block, ref cmt);
//cmt = parser.cmt;
}
//if (text_[i - 1].Block.mRule != null && id != text_[i - 1].Block.mRule.id) {
// return false;
//}
// コメントアウト状態に変化がなかったらここでお終い。
//if (i == tln() || text_[i].Block.isLineHeadCmt == cmt)
//if (i == tln() || (text_[i].Block.isLineHeadCmt == cmt && text_[i].Block.mRule == block.mRule)) {
//if (i == tln() || (text_[i].Block.isLineHeadCmt == cmt && text_[i].Block.mRule == block.mRule)
// || (text_[i].Block.mRule != null && id != text_[i].Block.mRule.id)) {
int tln = text.Count();
if (id != Document.DEFAULT_ID) {
tln = lines(text, ad, s, e);
//ll++;
//int ii = 0;
}
if (i == tln || (text[i].Block.isLineHeadCmt == cmt && text[i].Block.mRule == block.mRule)) {
//if (i == tln() || (text_[i].Block.isLineHeadCmt == cmt && text_[i].Block.mRule == block.mRule)) {
//var lists = lines(s, tln()==ll?e:ll);
var lists = lines(text, s, tln);
foreach (var list in lists) {
ParsePart(text, list);
}
return false;
}
int pcmt = 0;
Rule prule = null;
// 例えば、/* が入力された場合などは、下の方の行まで
// コメントアウト状態の変化を伝達する必要がある。
do {
Line line = text[i++];
pcmt = line.Block.isLineHeadCmt;
prule = line.Block.mRule;
if (i == tln) {
block = ParseLine(id, line, block, ref cmt, 0, las, true);
}
else {
block = ParseLine(id, line, block, ref cmt);
}
//cmt = parser.cmt;
if (pcmt == cmt) {
if (prule != block.mRule) {
pcmt--;
}
}
//} while (i < tln() && pcmt != cmt);
} while (i < tln && pcmt != cmt);
int ss = s;
int se = i;
return true;
}
private void lexSymbol(Block curblock)
{
StringBuilder buf = new StringBuilder();
int offset = reader.offset() - 1;
while (true) {
int c = reader.read();
if (c == -1) {
return;
}
buf.Append((char)c);
string s = buf.ToString();
if (ruleDic.ContainsKey(s)) {
var rule = ruleDic[s];
if (rule.Detected(s, reader)) {
tok = rule.token;
int len = rule.getLen(s, reader);
OffsetLenAttr = new Tuple<int, int, Attribute>(offset, len, rule.attr);
reader.setoffset(offset + len);
if (rule is MultiLineRule) {
if (curblock != null) {
curblock.mRule = rule as MultiLineRule;
//isNextLine = true;
}
}
break;
}
}
else if (multiRuleEndDic.ContainsKey(s)) {
var rule = multiRuleEndDic[s];
if (rule.Detected(s, reader)) {
tok = rule.token;
int len = rule.getLen(s, reader);
OffsetLenAttr = new Tuple<int, int, Attribute>(offset, len, rule.attr);
reader.setoffset(offset + len);
if (curblock != null) {
curblock.mRule = rule as MultiLineRule;
}
break;
}
}
}
}
public bool advance(Block preblock, Block curblock)
{
tok = TokenType.TXT;
if (Src.Length == 0 && curblock.isLineHeadCmt == 1) {
curblock.mRule = preblock.mRule;
isNextLine = true;
}
if (Src.Length == 0 && curblock.isLineHeadPart == 1) {
curblock.PartID = preblock.PartID;
scisNextLine = true;
}
int c = reader.read();
if (c == -1) {
tok = TokenType.EOS;
return false;
}
switch (c) {
case ' ':
case 0x3000:
case '\t':
break;
default:
//TODO test
if (curblock.isLineHeadCmt == 0) {
if (curblock.isLineHeadPart == 0) {
if (paruleStartKeys.Contains((char)c)) {
char fc = (char)c;
foreach (var item in partRuleDic) {
if (item.Key[0] == fc) {
var len = item.Value.start.Length;
if (Offset - 1 + len <= Src.Length) {
var text = Src.Substring(Offset - 1, len);
if (text.ToString() == item.Key) {
tok = TokenType.PartitionStart;
if (curblock != null) {
curblock.PartID = item.Value.id;
scisNextLine = true;
}
reader.setoffset(Offset + text.Length);
return true;
}
}
}
}
}
}else {
if (Offset - 1 == 0) {
while (c != -1) {
if (paruleStartKeys.Contains((char)c)) {
StringBuilder buf = new StringBuilder();
while (c != -1) {
buf.Append((char)c);
if (partRuleEndDic.ContainsKey(buf.ToString())) {
if (preblock.PartID == partRuleEndDic[buf.ToString()].id) {
var Eenelem = partRuleEndDic[buf.ToString()];
tok = TokenType.PartitionEnd;
curblock.PartID = Eenelem.id;
scisNextLine = false;
reader.setoffset(Offset + buf.ToString().Length);
return true;
}
}
c = reader.read();
}
}
c = reader.read();
}
//Finish:
if (c == -1) {
tok = TokenType.Partition;
curblock.PartID = preblock.PartID;
scisNextLine = true;
}
reader.setoffset(0);
c = reader.read();
}
else if (paruleStartKeys.Contains((char)c)) {
char fc = (char)c;
foreach (var item in partRuleDic) {
if (item.Key[0] == fc) {
var len = item.Value.start.Length;
if (Offset - 1 + len <= Src.Length) {
var text = Src.Substring(Offset - 1, len);
if (text.ToString() == item.Key) {
tok = TokenType.PartitionStart;
if (curblock != null) {
curblock.PartID = item.Key;
scisNextLine = true;
}
reader.setoffset(Offset + text.Length);
//break;
return true;
}
}
}
}
}
}
}
if (curblock.isLineHeadCmt == 0) { //0: 行頭がブロックコメントの内部ではない
if (Char.IsDigit((char)c)) {
reader.unread();
lexDigit();
break;
}
else if (Util.isIdentifierPart((char)c)) {
reader.unread();
lexKeyWord();
break;
}
else {
reader.unread();
lexSymbol(curblock);
break;
}
}
else { //1: 行頭がブロックコメントの内部
if (Offset - 1 == 0) {
reader.unread();
StringBuilder buf = new StringBuilder();
while (true) {
c = reader.read();
if (c == -1) {
break;
}
buf.Append((char)c);
string s = buf.ToString();
string end = preblock.mRule.end;
if (s.EndsWith(end)) {
if (multiRuleEndDic.ContainsKey(end)) {
var rule = multiRuleEndDic[end];
if (rule.Detected(end, reader)) {
curblock.mRule = rule;
//isNextLine = false;
tok = rule.token;
int len = rule.getLen(end, reader);
reader.setoffset(len);
OffsetLenAttr = new Tuple<int, int, Attribute>(0, len, rule.attr);
//break;
return true;
}
}
}
}
if (c == -1 && preblock.mRule != null && multiRuleDic.ContainsKey(preblock.mRule.start)) {
var enelem = multiRuleDic[preblock.mRule.start];
curblock.mRule = preblock.mRule;
tok = TokenType.MultiLineStart;
OffsetLenAttr = new Tuple<int, int, Attribute>(0, Src.Length, enelem.attr);
reader.setoffset(Src.Length);
}
break;
}
else {
if (Char.IsDigit((char)c)) {
reader.unread();
lexDigit();
}
else if (Util.isIdentifierPart((char)c)) {
reader.unread();
lexKeyWord();
}
else {
reader.unread();
lexSymbol(curblock);
}
}
}
break;
}
return true;
}
public bool advance(Block preblock, Block curblock)
{
tok = TokenType.TXT;
if (Src.Length == 0 && curblock.isLineHeadCmt == 1) {
curblock.mRule = preblock.mRule;
tok = TokenType.MultiLineAllLine;
isNextLine = true;
return true;
}
int c = reader.read();
if (c == -1) {
tok = TokenType.EOS;
return false;
}
switch (c) {
case ' ':
case 0x3000:
case '\t':
break;
default:
if (curblock.isLineHeadCmt == 0) { //0: 行頭がブロックコメントの内部ではない
if (Char.IsDigit((char)c)) {
reader.unread();
lexDigit();
break;
}
else if (Util.isIdentifierPart((char)c)) {
reader.unread();
lexKeyWord();
break;
}
else {
reader.unread();
lexSymbol(curblock);
break;
}
}
else { //1: 行頭がブロックコメントの内部
if (Offset - 1 == 0) {
int index;
if ((index = Src.IndexOf(preblock.mRule.end, 0)) >= 0) {
string end = preblock.mRule.end;
var rule = multiRuleEndDic[end];
if (rule.Detected(end, reader)) {
curblock.mRule = rule;
tok = TokenType.MultiLineEnd;
//int len = rule.getLen(end, reader);
int len = index + end.Length;
reader.setoffset(len);
OffsetLenAttr = new Tuple<int, int, Rule>(0, len, rule);
return true;
}
}
else if(multiRuleDic.ContainsKey(preblock.mRule.start)){
var rule = multiRuleDic[preblock.mRule.start];
curblock.mRule = preblock.mRule;
tok = TokenType.MultiLineAllLine;
OffsetLenAttr = new Tuple<int, int, Rule>(0, Src.Length, rule);
reader.setoffset(Src.Length);
return true;
}
//{
// reader.unread();
// StringBuilder buf = new StringBuilder();
// while (true) {
// c = reader.read();
// if (c == -1) {
// break;
// }
// buf.Append((char)c);
// string s = buf.ToString();
// string end = preblock.mRule.end;
// if (s.EndsWith(end)) {
// if (multiRuleEndDic.ContainsKey(end)) {
// var rule = multiRuleEndDic[end];
// if (rule.Detected(end, reader)) {
// curblock.mRule = rule;
// //isNextLine = false;
// //tok = rule.token;
// tok = TokenType.MultiLineEnd;
// int len = rule.getLen(end, reader);
// reader.setoffset(len);
// OffsetLenAttr = new Tuple<int, int, Rule>(0, len, rule);
// //break;
// return true;
// }
// }
// }
// }
// if (c == -1 && preblock.mRule != null && multiRuleDic.ContainsKey(preblock.mRule.start)) {
// var enelem = multiRuleDic[preblock.mRule.start];
// curblock.mRule = preblock.mRule;
// //isNextLine = true;
// //tok = TokenType.MultiLineStart;
// tok = TokenType.MultiLineAllLine;
// OffsetLenAttr = new Tuple<int, int, Rule>(0, Src.Length, enelem);
// reader.setoffset(Src.Length);
// }
// break;
//}
}
else {
if (Char.IsDigit((char)c)) {
reader.unread();
lexDigit();
}
else if (Util.isIdentifierPart((char)c)) {
reader.unread();
lexKeyWord();
}
else {
reader.unread();
lexSymbol(curblock);
}
}
}
break;
}
return true;
}