public void Reduce(IProduction production, ReduceVisitor reducer, Lexeme lookahead)
{
#if HISTORY
var from = m_topStack.StateIndex;
#endif
if (production == null)
{
//Accept
Debug.Assert(m_topStack.PrevNode.StateIndex == 0);
//TODO: accepted
IsAccepted = true;
return;
}
if (production.AggregatesAmbiguities)
{
AmbiguityAggregator = (production as ProductionBase).CreateAggregator();
}
var reduceResult = production.Accept(reducer, m_topStack);
m_topStack = reduceResult.NewTopStack;
var reduceError = reduceResult.ReduceError;
if (reduceError != null)
{
IncreaseErrorRecoverLevel();
if (reduceError.ErrorPosition == null)
{
reduceError.ErrorPosition = lookahead.Value.Span;
}
AddError(reduceError);
}
#if HISTORY
var to = m_topStack.StateIndex;
History.Add(String.Format("R{0}:{1}", from, to));
#endif
}
private TokenDecl TokenDeclaration()
{
var names = new List <Lexeme>();
Lexeme n = lexeme();
names.Add(n);
while (LookAhead(EngageToken.COMMA))
{
match(EngageToken.COMMA);
n = lexeme();
names.Add(n);
}
match(EngageToken.IS_TYPE);
return(new TokenDecl {
Names = names, Type = consumeText(EngageToken.ID)
});
}
public void OpenInLexicon(Lexeme lexeme)
{
string guid = null, toolName;
using (m_activationContext.Activate())
{
if (lexeme.Type == LexemeType.Word)
{
toolName = "Analyses";
IWfiWordform wf;
if (TryGetWordform(lexeme.LexicalForm, out wf))
{
guid = wf.Guid.ToString();
}
}
else
{
toolName = "lexiconEdit";
var entryLexeme = (FdoLexEntryLexeme)lexeme;
ILexEntry entry;
if (TryGetEntry(entryLexeme.Key, out entry))
{
guid = entry.Guid.ToString();
}
}
}
if (guid != null)
{
string url = string.Format("silfw://localhost/link?app=flex&database={0}&tool={1}&guid={2}",
HttpUtility.UrlEncode(m_cache.ProjectId.Name), HttpUtility.UrlEncode(toolName), HttpUtility.UrlEncode(guid));
// TODO: this would probably be faster if we directly called the RPC socket if FW is already open
if (MiscUtils.IsUnix)
{
string libPath = Path.GetDirectoryName(FileUtils.StripFilePrefix(Assembly.GetExecutingAssembly().CodeBase));
using (Process.Start(Path.Combine(libPath, "run-app"), string.Format("FieldWorks.exe {0}", url))) {}
}
else
{
using (Process.Start(url)) {}
}
}
}
public void TermWithField()
{
QueryLexer lexer = Lex("title:foo");
Assert.Equal(2, lexer.Lexemes.Count);
Lexeme fieldLexeme = lexer.Lexemes[0];
Assert.Equal(LexemeType.Field, fieldLexeme.Type);
Assert.Equal("title", fieldLexeme.Value);
Assert.Equal(0, fieldLexeme.Start);
Assert.Equal(5, fieldLexeme.End);
Lexeme termLexeme = lexer.Lexemes[1];
Assert.Equal(LexemeType.Term, termLexeme.Type);
Assert.Equal("foo", termLexeme.Value);
Assert.Equal(6, termLexeme.Start);
Assert.Equal(9, termLexeme.End);
}
private bool Take(LexemeKind lexemeKind, out Lexeme lexeme, bool ignoreWhitespace = true)
{
Push();
if (ignoreWhitespace)
{
SkipWhitespaces();
}
if (Current.Kind == lexemeKind)
{
lexeme = TakeAny();
Pop(false);
return(true);
}
lexeme = null;
Pop();
return(false);
}
public void MultipleCreatesReferToSameSenses()
{
Lexeme lex = m_lexicon.CreateLexeme(LexemeType.Word, "a");
Lexeme lex2 = m_lexicon.CreateLexeme(LexemeType.Word, "a");
m_lexicon.AddLexeme(lex);
LexiconSense sense = lex.AddSense();
sense.AddGloss("en", "test");
Assert.AreEqual(1, lex2.Senses.Count());
// Make sure the one that was added has the right sense now
lex = m_lexicon[lex.Id];
Assert.AreEqual(LexemeType.Word, lex.Type);
Assert.AreEqual("a", lex.LexicalForm);
Assert.AreEqual(1, lex.Senses.Count());
Assert.AreEqual("en", lex.Senses.First().Glosses.First().Language);
Assert.AreEqual("test", lex.Senses.First().Glosses.First().Text);
}
private static bool parserDefinition()
{
current_lexeme = Lexer.getDefinitionLexeme();
switch (current_lexeme)
{
case Lexeme.NAME:
if (!withdrawLexem(Lexeme.NAME) ||
!withdrawLexem(Lexeme.COLON) ||
!parserValue() ||
!parserProperties())
{
return(false);
}
break;
default:
return(true);
}
return(parserDefinition());
}
public void RemoveSenseSucceeds()
{
Lexeme lex = m_lexicon.CreateLexeme(LexemeType.Word, "a");
m_lexicon.AddLexeme(lex);
LexiconSense sense = lex.AddSense();
sense.AddGloss("en", "gloss1");
LexiconSense sense2 = lex.AddSense();
sense.AddGloss("en", "gloss1");
// Test remove at
lex.RemoveSense(sense2);
Assert.AreEqual(1, lex.Senses.Count());
Assert.AreEqual(sense, lex.Senses.First());
}
public void MultipleTermsProduceTwoLexemes()
{
QueryLexer lexer = Lex("foo bar");
Assert.Equal(2, lexer.Lexemes.Count);
Lexeme fooLexeme = lexer.Lexemes[0];
Assert.Equal(LexemeType.Term, fooLexeme.Type);
Assert.Equal("foo", fooLexeme.Value);
Assert.Equal(0, fooLexeme.Start);
Assert.Equal(3, fooLexeme.End);
Lexeme barLexeme = lexer.Lexemes[1];
Assert.Equal(LexemeType.Term, barLexeme.Type);
Assert.Equal("bar", barLexeme.Value);
Assert.Equal(4, barLexeme.Start);
Assert.Equal(7, barLexeme.End);
}
public void TermWithBoost()
{
QueryLexer lexer = Lex("foo^10");
Assert.Equal(2, lexer.Lexemes.Count);
Lexeme termLexeme = lexer.Lexemes[0];
Assert.Equal(LexemeType.Term, termLexeme.Type);
Assert.Equal("foo", termLexeme.Value);
Assert.Equal(0, termLexeme.Start);
Assert.Equal(3, termLexeme.End);
Lexeme boostLexeme = lexer.Lexemes[1];
Assert.Equal(LexemeType.Boost, boostLexeme.Type);
Assert.Equal("10", boostLexeme.Value);
Assert.Equal(4, boostLexeme.Start);
Assert.Equal(6, boostLexeme.End);
}
public void TermWithEditDistance()
{
QueryLexer lexer = Lex("foo~2");
Assert.Equal(2, lexer.Lexemes.Count);
Lexeme termLexeme = lexer.Lexemes[0];
Assert.Equal(LexemeType.Term, termLexeme.Type);
Assert.Equal("foo", termLexeme.Value);
Assert.Equal(0, termLexeme.Start);
Assert.Equal(3, termLexeme.End);
Lexeme editDistanceLexeme = lexer.Lexemes[1];
Assert.Equal(LexemeType.EditDistance, editDistanceLexeme.Type);
Assert.Equal("2", editDistanceLexeme.Value);
Assert.Equal(4, editDistanceLexeme.Start);
Assert.Equal(5, editDistanceLexeme.End);
}
public void TermWithPresenceProhibited()
{
QueryLexer lexer = Lex("-foo");
Assert.Equal(2, lexer.Lexemes.Count);
Lexeme presenceLexeme = lexer.Lexemes[0];
Assert.Equal(LexemeType.Presence, presenceLexeme.Type);
Assert.Equal("-", presenceLexeme.Value);
Assert.Equal(0, presenceLexeme.Start);
Assert.Equal(1, presenceLexeme.End);
Lexeme termLexeme = lexer.Lexemes[1];
Assert.Equal(LexemeType.Term, termLexeme.Type);
Assert.Equal("foo", termLexeme.Value);
Assert.Equal(1, termLexeme.Start);
Assert.Equal(4, termLexeme.End);
}
public void SingleTermWithHyphenProducesTwoLexemes()
{
QueryLexer lexer = Lex("foo-bar");
Assert.Equal(2, lexer.Lexemes.Count);
Lexeme fooLexeme = lexer.Lexemes[0];
Assert.Equal(LexemeType.Term, fooLexeme.Type);
Assert.Equal("foo", fooLexeme.Value);
Assert.Equal(0, fooLexeme.Start);
Assert.Equal(3, fooLexeme.End);
Lexeme barLexeme = lexer.Lexemes[1];
Assert.Equal(LexemeType.Term, barLexeme.Type);
Assert.Equal("bar", barLexeme.Value);
Assert.Equal(4, barLexeme.Start);
Assert.Equal(7, barLexeme.End);
}
public void TermWithFieldWithEscapeChar()
{
QueryLexer lexer = Lex(@"ti\:tle:foo");
Assert.Equal(2, lexer.Lexemes.Count);
Lexeme fieldLexeme = lexer.Lexemes[0];
Assert.Equal(LexemeType.Field, fieldLexeme.Type);
Assert.Equal("ti:tle", fieldLexeme.Value);
Assert.Equal(0, fieldLexeme.Start);
Assert.Equal(7, fieldLexeme.End);
Lexeme termLexeme = lexer.Lexemes[1];
Assert.Equal(LexemeType.Term, termLexeme.Type);
Assert.Equal("foo", termLexeme.Value);
Assert.Equal(8, termLexeme.Start);
Assert.Equal(11, termLexeme.End);
}
private VariableInfo ResolveVariable(Lexeme identifier)
{
//step1, check local parameter & variable definitions
if (m_currentMethodParameters.Contains(identifier.Value))
{
return(m_currentMethodParameters[identifier.Value]);
}
else if (m_currentMethodVariables.Contains(identifier.Value))
{
return(m_currentMethodVariables[identifier.Value]);
}
//step2, if not static method, check fields
if (!m_currentMethod.IsStatic)
{
return(ResolveField(m_currentType, identifier));
}
m_errorManager.AddError(c_SE_VariableDeclMissing, identifier.Span, identifier.Value);
return(null);
}
protected void processRightParenthesisLexeme(Lexeme rightParen)
{
while (true)
{
Lexeme popped;
try
{
popped = operatorStack.Pop();
}
catch (InvalidOperationException ex)
{
throw new ParseException("Mismatched parentheses!");
}
if (popped.Type == Lexeme.LexemeType.LEFT_PAREN)
{
try
{
// Ako je na vrhu stoga ostalo ime funkcije, prebacujemo ga u izlaz
IToken stackTop = operatorStack.Peek();
if (stackTop is Lexeme && properties.IsFunctionName((stackTop as Lexeme).Value))
{
Lexeme funcName = stackTop as Lexeme;
output.Enqueue(funcName);
operatorStack.Pop();
openedArgumentLists--;
}
}
catch (InvalidOperationException ex)
{
// do nothing, operator stack remains empty
}
return;
}
else
{
output.Enqueue(popped);
}
}
}
public override Parse <TFuture> BuildParse <TFuture>(Future <string, TFuture> future)
{
Parse <TFuture> parse = null;
parse = scanner =>
{
Lexeme l = scanner.Read();
if (l.IsEndOfStream)
{
return(new StepResult <TFuture>(0, () => future(l.Value.Content)(scanner)));
}
return(new StepResult <TFuture>(
1,
() => parse(scanner),
Grammar.RecoverByDeletion(l)));
};
return(parse);
}
/********************************************************************
*** FUNCTION ProcessDoubleToken ***
*********************************************************************
*** DESCRIPTION : This function is reponsible to categorize all the**
*** double operators like relational and arithmetic operators ***
*** INPUT ARGS : - ***
*** OUTPUT ARGS : - ***
*** IN/OUT ARGS : - ***
*** RETURN : void ***
********************************************************************/
private static void ProcessDoubleToken()
{
Lexeme = string.Concat(Lexeme, ch);
if (Lexeme.Equals("==") || Lexeme.Equals("!=") || Lexeme.Equals("<=") || Lexeme.Equals(">="))
{
Token = Symbol.relop;
}
if (Lexeme.Equals("||"))
{
Token = Symbol.addop;
}
if (Lexeme.Equals("&&"))
{
Token = Symbol.mulop;
}
GetNextCh();
}
public void MorphTypeRetained()
{
Lexeme lex = m_lexicon.CreateLexeme(LexemeType.Word, "a");
m_lexicon.AddLexeme(lex);
Lexeme lex2 = m_lexicon.CreateLexeme(LexemeType.Prefix, "a");
m_lexicon.AddLexeme(lex2);
Lexeme lex3 = m_lexicon.CreateLexeme(LexemeType.Suffix, "a");
m_lexicon.AddLexeme(lex3);
Lexeme lex4 = m_lexicon.CreateLexeme(LexemeType.Stem, "a");
m_lexicon.AddLexeme(lex4);
Assert.AreEqual(4, m_lexicon.Lexemes.Count());
Assert.IsTrue(m_lexicon.Lexemes.Contains(lex));
Assert.IsTrue(m_lexicon.Lexemes.Contains(lex2));
Assert.IsTrue(m_lexicon.Lexemes.Contains(lex3));
Assert.IsTrue(m_lexicon.Lexemes.Contains(lex4));
}
public void NormalizeStrings()
{
Lexeme lex = m_lexicon.CreateLexeme(LexemeType.Stem, "Vacaci\u00f3n"); // Uses composed accented letter 'o'
m_lexicon.AddLexeme(lex);
lex = m_lexicon[new LexemeKey(LexemeType.Stem, "Vacaci\u00f3n").Id];
Assert.IsNotNull(lex);
Assert.AreEqual(LexemeType.Stem, lex.Type);
Assert.AreEqual("Vacaci\u00f3n", lex.LexicalForm);
LexiconSense sense = lex.AddSense();
Assert.IsNotNull(sense);
LanguageText gloss = sense.AddGloss("en", "D\u00f3nde");
Lexeme reGetLex = m_lexicon[lex.Id];
Assert.AreEqual(gloss.Text, reGetLex.Senses.First().Glosses.First().Text);
}
public Token ProcessState(InputStream pInput, Lexeme pLexeme)
{
if (char.IsDigit(pInput.CurrentSymbol))
{
pLexeme.addSymbol(pInput.CurrentSymbol);
pInput.ConsumeSymbol();
return(new StateDigit().ProcessState(pInput, pLexeme));
}
else if (pInput.CurrentSymbol == '.')
{
pLexeme.addSymbol(pInput.CurrentSymbol);
pInput.ConsumeSymbol();
return(new StateFloat1().ProcessState(pInput, pLexeme));
}
else
{
return(new Token {
Type = TokenType.LitInteger, LexemeVal = pLexeme.Value,
Column = pInput.Column, Row = pInput.Row
});
}
}
public void MultipleTermsWithPresence()
{
QueryLexer lexer = Lex("+foo +bar");
Assert.Equal(4, lexer.Lexemes.Count);
Lexeme fooPresenceLexeme = lexer.Lexemes[0];
Lexeme fooLexeme = lexer.Lexemes[1];
Assert.Equal(LexemeType.Presence, fooPresenceLexeme.Type);
Assert.Equal(LexemeType.Term, fooLexeme.Type);
Assert.Equal("+", fooPresenceLexeme.Value);
Assert.Equal("foo", fooLexeme.Value);
Lexeme barPresenceLexeme = lexer.Lexemes[2];
Lexeme barLexeme = lexer.Lexemes[3];
Assert.Equal(LexemeType.Presence, barPresenceLexeme.Type);
Assert.Equal(LexemeType.Term, barLexeme.Type);
Assert.Equal("+", barPresenceLexeme.Value);
Assert.Equal("bar", barLexeme.Value);
}
private static bool parserProperties()
{
current_lexeme = Lexer.getPropertiesLexeme();
switch (current_lexeme)
{
case Lexeme.SEMICOLON:
if (!withdrawLexem(Lexeme.SEMICOLON))
{
return(false);
}
return(true);
default:
if (Lexer.checkForWhitespace() ||
!parserValue() ||
!parserProperties())
{
return(false);
}
return(true);
}
}
public void UpdateLexim()
{
IThemeGateway themeGateway = Gateway.ThemeGateway;
themeGateway.DeleteAllThemes();
Theme theme = new Theme(0, "Animals");
int id = themeGateway.AddTheme(theme);
ILexemGateway lexemGateway = Gateway.LexemGateway;
Lexeme lex = Lexemes.LexemeCollection[0];
lex.ParentThemeID = id;
lex.ID = lexemGateway.AddLexeme(lex);
lex.Word = "Fossil";
lexemGateway.UpdateLexeme(lex);
Lexeme lex1 = lexemGateway.GetLexeme(lex.ID);
Assert.AreEqual("Fossil", lex1.Word);
Assert.AreEqual(lex.ID, lex1.ID);
}
void ThemeView_OnSaveLexim(LeximDTView leximDT)
{
try
{
Lexeme lexim = new Lexeme(leximDT.ID, leximDT.ParentThemeID, leximDT.Word, leximDT.Picture, leximDT.Sound);
if (lexim.ID == 0)
{
leximDT.ID = LexemGateway.AddLexeme(lexim);
this.ThemeView.LeximListBox.InsertLexim(0, leximDT, this.ThemeView.Play);
this.ThemeView.LeximListBox.SetSelectionAt(0);
}
else
{
LexemGateway.UpdateLexeme(lexim);
this.ThemeView.LeximListBox.UpdateLexim(leximDT);
}
}
catch (Exception ex)
{
this.HandleException("Could not add or update a word ", ex);
}
}
private Expression Assignment()
{
Expression expr = Or();
if (Match(LexemeType.EQUAL))
{
Lexeme equals = Previous();
Expression value = Assignment();
if (expr is Expression.Variable &&
((Expression.Variable)expr) != null &&
((Expression.Variable)expr).Name != null)
{
Expression.Variable variable = (Expression.Variable)expr;
if (variable != null && variable.Name != null)
{
return(new Expression.Assign(variable.Name, value));
}
//> Classes assign-set
}
if (expr is Expression.Get &&
((Expression.Get)expr) != null &&
((Expression.Get)expr).Name != null)
{
Expression.Get get = (Expression.Get)expr;
if (get != null && get.Name != null)
{
return(new Expression.Set(get, get.Name, value));
}
//< Classes assign-set
}
//error(equals, "Invalid assignment target."); // [no-throw]
}
return(expr);
}
public override Parse <TFuture> BuildParse <TFuture>(Future <string, TFuture> future)
{
Parse <TFuture> parse = null;
parse = scanner =>
{
// 保存读取当前单词之前的位置
ForkableScanner prevScanner = scanner.Fork();
Lexeme l = scanner.Read();
if (l.TokenIndex == m_token.Index)
{
return(new StepResult <TFuture>(0, () => future(l.Value.Content)(scanner)));
}
Lexeme recovery = l.GetErrorCorrectionLexeme(m_token.Index, m_token.Description);
SyntaxError insertionErr = Grammar.RecoverByInsertion(recovery);
if (l.IsEndOfStream)
{
// 已经到了输入的末尾
// 插入预期的Token进行恢复
return(new StepResult <TFuture>(1,
() => future(recovery.Value.Content)(prevScanner), insertionErr));
}
else
{
// 同时尝试插入预期的Token，以及删除当前字符
// 在未来的解析中选取更好的路线
return(Grammar.Best(
new StepResult <TFuture>(1, () => future(recovery.Value.Content)(prevScanner), insertionErr),
new StepResult <TFuture>(1, () => parse(scanner), Grammar.RecoverByDeletion(l))));
}
};
return(parse);
}
public static LinkedListNode <Lexeme> TryParse(LinkedListNode <Lexeme> lexemes, out FnParam resultNode)
{
resultNode = null;
var nextLexeme = Type.TryParse(lexemes, out Type type);
if (type == null)
{
return(lexemes);
}
if (nextLexeme.Value.Type != LT.IDENT)
{
throw new Exception($"Missing function argument name at {nextLexeme.Value.File}:{nextLexeme.Value.Line}");
}
Lexeme token = nextLexeme.Value;
nextLexeme = nextLexeme.Next;
if (nextLexeme.Value.Type == LT.OP_ASSIGN)
{
nextLexeme = nextLexeme.Next;
nextLexeme = Literal.TryParse(nextLexeme, out Literal lit);
if (lit == null)
{
throw new Exception($"Missing default value after assignment operator in function arguments list at {nextLexeme.Value.File}:{nextLexeme.Value.Line}");
}
resultNode = new FnParam {
Type = type, DefaultValue = lit, Token = token
};
return(nextLexeme);
}
resultNode = new FnParam {
Type = type, Token = token
};
return(nextLexeme);
}
private Expression FinishCall(Expression callee)
{
List <Expression> arguments = new List <Expression>();
if (!Check(LexemeType.RIGHT_PAREN))
{
do
{
////> check-max-arity
//if (arguments.size() >= 255)
//{
// error(peek(), "Can't have more than 255 arguments.");
//}
////< check-max-arity
arguments.Add(Expr());
} while (Match(LexemeType.COMMA));
}
Lexeme paren = Consume(LexemeType.RIGHT_PAREN, Peek());
return(new Expression.Call(callee, paren, arguments));
}
//private bool isSubscriptedIdentifier(SubscriptToken t)
//{
// TokenList subBase = (token as SubscriptToken).Base;
// TokenList subscript = (token as SubscriptToken).Subscript;
// if (subBase.Count == 1 && subBase[0] is TextRunToken)
// {
// String subBaseText = (subBase[0] as TextRunToken).Text;
// if (Regex.Match(subBaseText, )
// }
// // TODO : implementiraj!!!!
// return false;
//}
protected bool canProcessTokenAsUnaryOp()
{
if (lastProcessedElement == null)
{
return(true);
}
else if (lastProcessedElement is Lexeme)
{
Lexeme previous = lastProcessedElement as Lexeme;
Lexeme.LexemeType type = previous.Type;
return(type == Lexeme.LexemeType.LEFT_PAREN ||
type == Lexeme.LexemeType.EQ_SIGN ||
type == Lexeme.LexemeType.OP_DIV ||
type == Lexeme.LexemeType.OP_MUL ||
type == Lexeme.LexemeType.OP_MINUS ||
type == Lexeme.LexemeType.OP_PLUS ||
type == Lexeme.LexemeType.ARGUMENT_SEPARATOR);
}
else
{
return(false);
}
}
/// <summary>*
/// 处理数字字母混合输出
/// 如：windos2000 | [email protected]
/// </summary>
private bool ProcessMixLetter(char input, Context context)
{
bool needLock = false;
if (start == -1)
{//当前的分词器尚未开始处理字符
if (IsAcceptedCharStart(input))
{
//记录起始指针的位置,标明分词器进入处理状态
start = context.Cursor;
end = start;
}
}
else
{//当前的分词器正在处理字符
if (IsAcceptedChar(input))
{
//输入不是连接符
// if(!isLetterConnector(input)){
//记录下可能的结束位置，如果是连接符结尾，则忽略
// end = context.getCursor();
// }
//不在忽略尾部的链接字符
end = context.Cursor;
}
else
{
//生成已切分的词元
Lexeme newLexeme = new Lexeme(context.BuffOffset, start, end - start + 1, Lexeme.TYPE_LETTER);
context.AddLexeme(newLexeme);
//设置当前分词器状态为“待处理”
start = -1;
end = -1;
}
}
//context.getCursor() == context.getAvailable() - 1读取缓冲区最后一个字符，直接输出
if (context.Cursor == context.Available - 1)
{
if (start != -1 && end != -1)
{
//生成已切分的词元
Lexeme newLexeme = new Lexeme(context.BuffOffset, start, end - start + 1, Lexeme.TYPE_LETTER);
context.AddLexeme(newLexeme);
}
//设置当前分词器状态为“待处理”
start = -1;
end = -1;
}
//判断是否锁定缓冲区
if (start == -1 && end == -1)
{
//对缓冲区解锁
needLock = false;
}
else
{
needLock = true;
}
return needLock;
}
public FdoLexiconGlossAddedEventArgs(Lexeme lexeme, LexiconSense sense, LanguageText gloss)
{
m_lexeme = lexeme;
m_sense = sense;
m_gloss = gloss;
}
public void CheckToken(Lexeme t)
{
Debug.Assert(Lexeme.Name <= t); // FirstStringable
if (lexeme != t)
{
if (t == Lexeme.Eof)
throw EofExpectedException(RawValue);
else
throw TokenExpectedException(t, RawValue);
}
}
/// <summary>
/// 取出链表集合的最后一个元素
/// </summary>
/// <returns></returns>
internal Lexeme PollLast()
{
if (this.size == 1)
{
Lexeme last = this.head;
this.head = null;
this.tail = null;
this.size--;
return last;
}
else if (this.size > 1)
{
Lexeme last = this.tail;
this.tail = last.Prev;
last.Prev = null;
this.size--;
return last;
}
else
{
return null;
}
}
/// <summary>
/// 向链表集合添加词元
/// </summary>
internal void AddLexeme(Lexeme lexeme)
{
if (this.size == 0)
{
this.head = lexeme;
this.tail = lexeme;
this.size++;
return;
}
else
{
if (this.tail.CompareTo(lexeme) == 0)
{//词元与尾部词元相同，不放入集合
return;
}
else if (this.tail.CompareTo(lexeme) < 0)
{//词元接入链表尾部
this.tail.Next = lexeme;
lexeme.Prev = this.tail;
this.tail = lexeme;
this.size++;
return;
}
else if (this.head.CompareTo(lexeme) > 0)
{//词元接入链表头部
this.head.Prev = lexeme;
lexeme.Next = this.head;
this.head = lexeme;
this.size++;
return;
}
else
{
//从尾部上逆
Lexeme l = this.tail;
while (l != null && l.CompareTo(lexeme) > 0)
{
l = l.Prev;
}
if (l.CompareTo(lexeme) == 0)
{//词元与集合中的词元重复，不放入集合
return;
}
else if (l.CompareTo(lexeme) < 0)
{//词元插入链表中的某个位置
lexeme.Prev = l;
lexeme.Next = l.Next;
l.Next.Prev = lexeme;
l.Next = lexeme;
this.size++;
return;
}
}
}
}
/// <summary>
/// 添加量词词元到结果集
/// </summary>
/// <param name="context"></param>
private void OutputCountLexeme(Context context)
{
if (countStart > -1 && countEnd > -1)
{
//生成已切分的词元
Lexeme countLexeme = new Lexeme(context.BuffOffset, countStart, countEnd - countStart + 1, Lexeme.TYPE_NUMCOUNT);
context.AddLexeme(countLexeme);
}
}
public void NextLex()
{
prevLexEnd = curIndex;
prevLexeme = lexeme;
SkipWhiteSpace();
lexStart = curIndex;
switch (curChar)
{
case '\0':
lexeme = Lexeme.Eof;
return;
case '(':
case ')':
case '[':
case ']':
case '@':
case ',':
case '$':
case '}':
lexeme = (Lexeme)curChar;
NextChar();
break;
case '.':
NextChar();
if (curChar == '.')
{
lexeme = Lexeme.DotDot;
NextChar();
}
else if (IsAsciiDigit(curChar))
{
SetSourceIndex(lexStart);
goto case '0';
}
else
lexeme = Lexeme.Dot;
break;
case ':':
NextChar();
if (curChar == ':')
{
lexeme = Lexeme.ColonColon;
NextChar();
}
else
{
lexeme = Lexeme.Unknown;
}
break;
case '*':
lexeme = Lexeme.Star;
NextChar();
CheckOperator(true);
break;
case '/':
NextChar();
if (curChar == '/')
{
lexeme = Lexeme.SlashSlash;
NextChar();
}
else
{
lexeme = Lexeme.Slash;
}
break;
case '|':
lexeme = Lexeme.Union;
NextChar();
break;
case '+':
lexeme = Lexeme.Plus;
NextChar();
break;
case '-':
lexeme = Lexeme.Minus;
NextChar();
break;
case '=':
lexeme = Lexeme.Eq;
NextChar();
break;
case '!':
NextChar();
if (curChar == '=')
{
lexeme = Lexeme.Ne;
NextChar();
}
else
{
lexeme = Lexeme.Unknown;
}
break;
case '<':
NextChar();
if (curChar == '=')
{
lexeme = Lexeme.Le;
NextChar();
}
else
{
lexeme = Lexeme.Lt;
}
break;
case '>':
NextChar();
if (curChar == '=')
{
lexeme = Lexeme.Ge;
NextChar();
}
else
{
lexeme = Lexeme.Gt;
}
break;
case '"':
case '\'':
lexeme = Lexeme.String;
ScanString();
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
lexeme = Lexeme.Number;
ScanNumber();
break;
default:
this.name = ScanNCName();
if (this.name != null)
{
lexeme = Lexeme.Name;
this.prefix = string.Empty;
this.canBeFunction = false;
this.axis = XPathAxis.Unknown;
bool colonColon = false;
int saveSourceIndex = curIndex;
// "foo:bar" or "foo:*" -- one lexeme (no spaces allowed)
// "foo::" or "foo ::" -- two lexemes, reported as one (AxisName)
// "foo:?" or "foo :?" -- lexeme "foo" reported
if (curChar == ':')
{
NextChar();
if (curChar == ':')
{ // "foo::" -> OperatorName, AxisName
NextChar();
colonColon = true;
SetSourceIndex(saveSourceIndex);
}
else
{ // "foo:bar", "foo:*" or "foo:?"
string ncName = ScanNCName();
if (ncName != null)
{
this.prefix = this.name;
this.name = ncName;
// Look ahead for '(' to determine whether QName can be a FunctionName
saveSourceIndex = curIndex;
SkipWhiteSpace();
this.canBeFunction = (curChar == '(');
SetSourceIndex(saveSourceIndex);
}
else if (curChar == '*')
{
NextChar();
this.prefix = this.name;
this.name = "*";
}
else
{ // "foo:?" -> OperatorName, NameTest
// Return "foo" and leave ":" to be reported later as an unknown lexeme
SetSourceIndex(saveSourceIndex);
}
}
}
else
{
SkipWhiteSpace();
if (curChar == ':')
{ // "foo ::" or "foo :?"
NextChar();
if (curChar == ':')
{
NextChar();
colonColon = true;
}
SetSourceIndex(saveSourceIndex);
}
else
{
this.canBeFunction = (curChar == '(');
}
}
if (!CheckOperator(false) && colonColon)
{
this.axis = CheckAxis();
}
}
else
{
lexeme = Lexeme.Unknown;
NextChar();
}
break;
}
}
private List<Lexeme> AllowedFollowing(Lexeme t)
{
var list = new List<Lexeme>();
switch (t)
{
case Lexeme.Axis:
list.Add(Lexeme.ColonColon);
return list;
case Lexeme.ColonColon:
list.Add(Lexeme.Name);
list.Add(Lexeme.Star);
return list;
case Lexeme.And:
list.Add(Lexeme.String);
return list;
case Lexeme.At:
list.Add(Lexeme.Name);
list.Add(Lexeme.Star);
return list;
case Lexeme.Name:
list.Add(Lexeme.Slash);
list.Add(Lexeme.LParens);
list.Add(Lexeme.LBracket);
return list;
case Lexeme.Slash:
list.Add(Lexeme.Name);
list.Add(Lexeme.Star);
return list;
case Lexeme.SlashSlash:
list.Add(Lexeme.Name);
list.Add(Lexeme.Star);
return list;
case Lexeme.Eq:
list.Add(Lexeme.Number);
list.Add(Lexeme.Name);
return list;
case Lexeme.LParens:
list.Add(Lexeme.RParens);
list.Add(Lexeme.Name);
return list;
}
return list;
}
private List<Lexeme> AllowedWhenExpected(Lexeme t)
{
var list = new List<Lexeme>();
list.Add(t);
switch (t)
{
case Lexeme.RBracket:
{
bool foundEq= false;
// last on the stack is first in the array
var x = lexemeStack.ToArray();
for (int i= 0; i < x.Length; i++)
{
if (x[i] == Lexeme.LBracket) break;
if (x[i] == Lexeme.Eq) { foundEq = true; break; }
}
if (!foundEq)
list.Add(Lexeme.Eq);
}
break;
}
return list;
}
private XPathAxis CheckAxis()
{
this.lexeme = Lexeme.Axis;
switch (name)
{
case "ancestor": return XPathAxis.Ancestor;
case "ancestor-or-self": return XPathAxis.AncestorOrSelf;
case "attribute": return XPathAxis.Attribute;
case "child": return XPathAxis.Child;
case "descendant": return XPathAxis.Descendant;
case "descendant-or-self": return XPathAxis.DescendantOrSelf;
case "following": return XPathAxis.Following;
case "following-sibling": return XPathAxis.FollowingSibling;
case "namespace": return XPathAxis.Namespace;
case "parent": return XPathAxis.Parent;
case "preceding": return XPathAxis.Preceding;
case "preceding-sibling": return XPathAxis.PrecedingSibling;
case "self": return XPathAxis.Self;
default:
this.lexeme = Lexeme.Name;
return XPathAxis.Unknown;
}
}
private bool CheckOperator(bool star)
{
Lexeme op;
if (star)
{
op = Lexeme.Multiply;
}
else
{
if (prefix.Length != 0 || name.Length > 3)
return false;
switch (name)
{
case "or": op = Lexeme.Or; break;
case "and": op = Lexeme.And; break;
case "div": op = Lexeme.Divide; break;
case "mod": op = Lexeme.Modulo; break;
default: return false;
}
}
// If there is a preceding token and the preceding token is not one of '@', '::', '(', '[', ',' or an Operator,
// then a '*' must be recognized as a MultiplyOperator and an NCName must be recognized as an OperatorName.
if (prevLexeme <= Lexeme.LastOperator)
return false;
switch (prevLexeme)
{
case Lexeme.Slash:
case Lexeme.SlashSlash:
case Lexeme.At:
case Lexeme.ColonColon:
case Lexeme.LParens:
case Lexeme.LBracket:
case Lexeme.Comma:
case Lexeme.Dollar:
return false;
}
this.lexeme = op;
return true;
}
public ExpressionLexeme(string lexemeBody, Lexeme lexeme)
{
_lexemeBody = lexemeBody;
_lexeme = lexeme;
}
private static LexemeType GetLexemeType(Lexeme lexeme)
{
if (ExpressionLexemeInfo.LexemeTypes.ContainsKey(lexeme))
return (LexemeType)ExpressionLexemeInfo.LexemeTypes[lexeme];
else
return LexemeType.ltUndefined;
}
// May be called for the following tokens: Name, String, Eof, Comma, LParens, RParens, LBracket, RBracket, RBrace
private string LexemeToString(Lexeme t)
{
Debug.Assert(Lexeme.Name <= t);
if (Lexeme.Eof < t)
{
Debug.Assert("()[].@,*/$}".IndexOf((char)t) >= 0);
return new string((char)t, 1);
}
switch (t)
{
case Lexeme.Name: return "<name>";
case Lexeme.String: return "<string literal>";
case Lexeme.Eof: return "<eof>";
default:
Debug.Fail("Unexpected Lexeme: " + t.ToString());
return string.Empty;
}
}
public XPathParserException TokenExpectedException(Lexeme t, string actualToken)
{
string expectedToken = LexemeToString(t);
var x = new XPathParserException(xpathExpr, lexStart, curIndex,
string.Format("Expected token '{0}', found '{1}'.", expectedToken, actualToken)
);
x.NextAllowed= AllowedWhenExpected(t);
return x;
}
public Rule(String regex, Lexeme.Type type)
: this(regex, DefaultProducer(type))
{
}
/// <summary>
/// 添加数词词元到结果集
/// </summary>
/// <param name="context"></param>
private void OutputNumLexeme(Context context)
{
if (nStart > -1 && nEnd > -1)
{
//生成已切分的词元
Lexeme newLexeme = new Lexeme(context.BuffOffset, nStart, nEnd - nStart + 1, Lexeme.TYPE_NUM);
context.AddLexeme(newLexeme);
fCaN = true;
}
}
private static Producer DefaultProducer(Lexeme.Type type)
{
return (value, pos) => new Lexeme(value, type, pos);
}
/// <summary>
/// 向分词结果集添加词元
/// </summary>
/// <param name="lexeme"></param>
public void AddLexeme(Lexeme lexeme)
{
if (!Dictionary.IsStopWord(segmentBuff, lexeme.Begin, lexeme.Length))
{
this.lexemeSet.AddLexeme(lexeme);
}
}
/// <summary>
/// 取出词元集合中的下一个词元
/// </summary>
/// <param name="lexeme"></param>
/// <returns></returns>
private Lexeme BuildLexeme(Lexeme lexeme)
{
if (lexeme != null)
{
// TODO:从数组取一段内容
var arr = new char[lexeme.Length];
Array.Copy(segmentBuff, lexeme.Begin, arr, 0, lexeme.Length);
//生成lexeme的词元文本
lexeme.LexemeText = string.Concat(arr);
return lexeme;
}
else
{
return null;
}
}
/// <summary>
/// 取出链表集合的第一个元素
/// </summary>
internal Lexeme PollFirst()
{
if (this.size == 1)
{
Lexeme first = this.head;
this.head = null;
this.tail = null;
this.size--;
return first;
}
else if (this.size > 1)
{
Lexeme first = this.head;
this.head = first.Next;
first.Next = null;
this.size--;
return first;
}
else
{
return null;
}
}
public TokenBranch(Lexeme lexeme)
{
if (lexeme != null)
{
this.lexeme = lexeme;
//初始化branch的左右边界
this.leftBorder = lexeme.BeginPosition;
this.rightBorder = lexeme.EndPosition;
}
}
/// <summary>
/// 向分词结果集添加词元（尚未完成）
/// </summary>
/// <param name="lexeme"></param>
public void AddLexeme(Lexeme lexeme)
{
}
///<summary>
///组合词元分支
/// </summary>
/// <returns>返回当前branch能否接收词元对象</returns>
public bool Accept(Lexeme _lexeme)
{
/*
* 检查新的lexeme 对当前的branch 的可接受类型
* acceptType : REFUSED 不能接受
* acceptType : ACCEPTED 接受
* acceptType : TONEXT 由相邻分支接受
*/
int acceptType = CheckAccept(_lexeme);
switch (acceptType)
{
case REFUSED:
// REFUSE 情况
return false;
case ACCEPTED:
if (acceptedBranchs == null)
{
//当前branch没有子branch，则添加到当前branch下
acceptedBranchs = new List<TokenBranch>(2);
acceptedBranchs.Add(new TokenBranch(_lexeme));
}
else
{
bool acceptedByChild = false;
//当前branch拥有子branch，则优先由子branch接纳
foreach (TokenBranch childBranch in acceptedBranchs)
{
acceptedByChild = childBranch.Accept(_lexeme) || acceptedByChild;
}
//如果所有的子branch不能接纳，则由当前branch接纳
if (!acceptedByChild)
{
acceptedBranchs.Add(new TokenBranch(_lexeme));
}
}
//设置branch的最大右边界
if (_lexeme.EndPosition > this.rightBorder)
{
this.rightBorder = _lexeme.EndPosition;
}
break;
case TONEXT:
//把lexeme放入当前branch的相邻分支
if (this.nextBranch == null)
{
//如果还没有相邻分支，则建立一个不交叠的分支
this.nextBranch = new TokenBranch(null);
}
this.nextBranch.Accept(_lexeme);
break;
}
return true;
}
public FdoLexemeAddedEventArgs(Lexeme lexeme)
{
m_lexeme = lexeme;
}
/// <summary>
/// 判断指定的lexeme能否被当前的branch接受
/// </summary>
/// <param name="_lexeme"></param>
/// <returns></returns>
private int CheckAccept(Lexeme _lexeme)
{
int acceptType = 0;
if (_lexeme == null)
{
throw new ArgumentException("parameter:lexeme is null");
}
if (null == this.lexeme)
{//当前的branch是一个不交叠（ROOT）的分支
if (this.rightBorder > 0 //说明当前branch内至少有一个lexeme
&& _lexeme.BeginPosition >= this.rightBorder)
{
//_lexeme 与 当前的branch不相交
acceptType = TONEXT;
}
else
{
acceptType = ACCEPTED;
}
}
else
{//当前的branch是一个有交叠的分支
if (_lexeme.BeginPosition < this.lexeme.BeginPosition)
{
//_lexeme 的位置比 this.lexeme还靠前（这种情况不应该发生）
acceptType = REFUSED;
}
else if (_lexeme.BeginPosition >= this.lexeme.BeginPosition
&& _lexeme.BeginPosition < this.lexeme.EndPosition)
{
// _lexeme 与 this.lexeme相交
acceptType = REFUSED;
}
else if (_lexeme.BeginPosition >= this.lexeme.EndPosition
&& _lexeme.BeginPosition < this.rightBorder)
{
//_lexeme 与 this.lexeme 不相交， 但_lexeme 与 当前的branch相交
acceptType = ACCEPTED;
}
else
{//_lexeme.BeginPosition >= this.rightBorder
//_lexeme 与 当前的branch不相交
acceptType = TONEXT;
}
}
return acceptType;
}
public FdoLexicalRelation(Lexeme lexeme, string name)
{
m_lexeme = lexeme;
m_name = name;
}
public void PassToken(Lexeme t)
{
CheckToken(t);
NextLex();
}
private static string GetLexemeDiffMask(Lexeme lexeme)
{
return (string)ExpressionLexemeInfo.LexemeDiffMasks[lexeme];
}
private static string GetLexemeCode(Lexeme lexeme)
{
return (string)ExpressionLexemeInfo.LexemeCodes[lexeme];
}