internal string GetPrefix(TextBuffer buffer)
{
return(prefixSpan.GetString(buffer));
}
//
public virtual void Lex(TextBuffer textBuffer)
{
WordSpanKind lexMode = WordSpanKind.Unknown;
//simple line break / whitespace lexer
char[] buffer = textBuffer.UnsafeGetInternalBuffer();
int j = buffer.Length;
int startIndex = 0;
int len = 0;
_spans.Clear();
for (int i = 0; i < j; ++i)
{
char c = buffer[i];
if (c == '\r')
{
if (len > 0)
{
//TODO: check if len > int?
var newspan = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan);
startIndex = i;
len = 0;
}
len++;
lexMode = WordSpanKind.NewLine;
//if next is '\n'
if (i < j - 1)
{
char c1 = buffer[i + 1];
if (c1 == '\n')
{
//push accum wordspan
//
//newline wordspan
i++;
len++;
var newspan = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan);
startIndex += len;
len = 0;
}
else
{
var newspan = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan);
startIndex += len;
len = 0;
}
}
else
{
//the \r is last char
var newspan = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan);
startIndex += len;
len = 0;
}
}
else if (c == '\n')
{
//new line?
if (len > 0)
{
//TODO: check if len > int?
var newspan = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan);
startIndex = i;
len = 0;
}
//
lexMode = WordSpanKind.NewLine;
var newspan2 = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan2);
len = 0;
startIndex = i;
}
else if (c == '\t')
{
//tab character
//
if (lexMode != WordSpanKind.Tab)
{
//flush
if (len > 0)
{
var newspan = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan);
startIndex = i;
len = 0;
}
}
len++;
lexMode = WordSpanKind.Tab;
}
else if (char.IsWhiteSpace(c))
{
//whitespace
if (lexMode != WordSpanKind.WhiteSpace)
{
//change mode
if (len > 0)
{
//TODO: check if len > int?
var newspan = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan);
startIndex = i;
len = 0;
}
}
len++;
lexMode = WordSpanKind.WhiteSpace;
}
else
{
if (lexMode != WordSpanKind.Text)
{
//change mode
if (len > 0)
{
//TODO: check if len > int?
var newspan = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan);
startIndex = i;
len = 0;
}
}
len++;
lexMode = WordSpanKind.Text;
}
}
//
//finish
if (len > 0)
{
var newspan = new LexWordSpan(startIndex,
(ushort)len, lexMode);
_spans.Add(newspan);
len = 0;
}
}
public void LoadFromTextfile(string filename)
{
//once only
if (textBuffer != null)
{
return;
}
if (firstChar == '\0' || lastChar == '\0')
{
throw new NotSupportedException();
}
//---------------
Dictionary <char, DevelopingWordGroup> wordGroups = new Dictionary <char, DevelopingWordGroup>();
using (FileStream fs = new FileStream(filename, FileMode.Open))
using (StreamReader reader = new StreamReader(fs))
{
//init with filesize
textBuffer = new TextBuffer((int)fs.Length);
string line = reader.ReadLine();
while (line != null)
{
line = line.Trim();
char[] lineBuffer = line.ToCharArray();
int lineLen = lineBuffer.Length;
char c0;
if (lineLen > 0 && (c0 = lineBuffer[0]) != '#')
{
int startAt = textBuffer.CurrentPosition;
textBuffer.AddWord(lineBuffer);
#if DEBUG
if (lineLen > byte.MaxValue)
{
throw new NotSupportedException();
}
#endif
WordSpan wordspan = new WordSpan(startAt, (byte)lineLen);
//each wordgroup contains text span
DevelopingWordGroup found;
if (!wordGroups.TryGetValue(c0, out found))
{
found = new DevelopingWordGroup(new WordSpan(startAt, 1));
wordGroups.Add(c0, found);
}
found.AddWordSpan(wordspan);
}
//- next line
line = reader.ReadLine();
}
reader.Close();
fs.Close();
}
//------------------------------------------------------------------
textBuffer.Freeze();
//------------------------------------------------------------------
//do index
DoIndex(wordGroups);
//clear, not used
wordGroups.Clear();
}
WordGroup _resultWordGroup;//after call DoIndex()
internal void DoIndex(TextBuffer textBuffer, CustomDic owner)
{
//recursive
if (this.PrefixLen > 7)
{
DoIndexOfSmallAmount(textBuffer);
#if DEBUG
dbugDataState = debugDataState.TooLongPrefix;
#endif
return;
}
//-----------------------------------------------
bool hasEvalPrefix = false;
if (subGroups == null)
{
subGroups = new DevelopingWordGroup[owner.LastChar - owner.FirstChar + 1];
}
//--------------------------------
int j = wordSpanList.Count;
int thisPrefixLen = this.PrefixLen;
int doSepAt = thisPrefixLen;
for (int i = 0; i < j; ++i)
{
WordSpan sp = wordSpanList[i];
if (sp.len > doSepAt)
{
char c = sp.GetChar(doSepAt, textBuffer);
int c_index = c - owner.FirstChar;
DevelopingWordGroup found = subGroups[c_index];
if (found == null)
{
//not found
found = new DevelopingWordGroup(new WordSpan(sp.startAt, (byte)(doSepAt + 1)));
subGroups[c_index] = found;
}
found.AddWordSpan(sp);
}
else
{
if (!hasEvalPrefix)
{
if (sp.SameTextContent(this.prefixSpan, textBuffer))
{
hasEvalPrefix = true;
this.PrefixIsWord = true;
}
}
}
}
#if DEBUG
this.dbugDataState = debugDataState.Indexed;
#endif
wordSpanList.Clear();
wordSpanList = null;
//--------------------------------
//do sup index
//foreach (WordGroup subgroup in this.wordGroups.Values)
bool hasSomeSubGroup = false;
foreach (DevelopingWordGroup subgroup in this.subGroups)
{
if (subgroup != null)
{
hasSomeSubGroup = true;
//****
//performance factor here,****
//in this current version
//if we not call DoIndex(),
//this subgroup need linear search-> so it slow
//so we call DoIndex until member count in the group <=3
//then it search faster,
//but dictionary-building time may increase.
if (subgroup.WordSpanListCount > 2)
{
subgroup.DoIndex(textBuffer, owner);
}
else
{
#if DEBUG
subgroup.dbugDataState = debugDataState.SmallAmountOfMembers;
#endif
subgroup.DoIndexOfSmallAmount(textBuffer);
}
}
}
//--------------------------------
#if DEBUG
this.dbugDataState = debugDataState.Indexed;
#endif
if (!hasSomeSubGroup)
{
//clear
subGroups = null;
}
//--------------------------------
WordGroup[] newsubGroups = null;
if (subGroups != null)
{
newsubGroups = new WordGroup[subGroups.Length];
for (int i = subGroups.Length - 1; i >= 0; --i)
{
DevelopingWordGroup subg = subGroups[i];
if (subg != null)
{
newsubGroups[i] = subg.ResultWordGroup;
}
}
}
//--------------------------------
this._resultWordGroup = new WordGroup(
this.prefixSpan,
newsubGroups,
null,
this.PrefixIsWord);
}
public string GetString(TextBuffer textBuffer)
{
return(textBuffer.GetString(startAt, len));
}
public char GetChar(int index, TextBuffer textBuffer)
{
return(textBuffer.GetChar(startAt + index));
}
int FindInWordSpans(WordVisitor visitor, WordGroup wordGroup)
{
WordSpan[] wordSpans = wordGroup.GetWordSpans();
if (wordSpans == null)
{
throw new NotSupportedException();
}
//at this wordgroup
//no subground anymore
//so we should find the word one by one
//start at prefix
//and select the one that
int readLen = visitor.CurrentIndex - visitor.LatestBreakAt;
int nwords = wordSpans.Length;
//only 1 that match
TextBuffer currentTextBuffer = CurrentCustomDic.TextBuffer;
//we sort unindex string ***
//so we find from longest one( last) to begin
for (int i = nwords - 1; i >= 0; --i)
{
//loop test on each word
WordSpan w = wordSpans[i];
#if DEBUG
//string dbugstr = w.GetString(currentTextBuffer);
#endif
int savedIndex = visitor.CurrentIndex;
char c = visitor.Char;
int wordLen = w.len;
int matchCharCount = 0;
if (wordLen > readLen)
{
for (int p = readLen; p < wordLen; ++p)
{
char c2 = w.GetChar(p, currentTextBuffer);
if (c2 == c)
{
matchCharCount++;
//match
//read next
if (!visitor.IsEnd)
{
visitor.SetCurrentIndex(visitor.CurrentIndex + 1);
c = visitor.Char;
}
else
{
//no more data in visitor
break;
}
}
else
{
break;
}
}
}
//reset
if (readLen + matchCharCount == wordLen)
{
int newBreakAt = visitor.LatestBreakAt + wordLen;
visitor.SetCurrentIndex(newBreakAt);
//--------------------------------------------
if (visitor.State == VisitorState.End)
{
return(newBreakAt);
}
//check next char can be the char of new word or not
//this depends on each lang
char canBeStartChar = visitor.Char;
if (CanHandle(canBeStartChar))
{
if (CanBeStartChar(canBeStartChar))
{
return(newBreakAt);
}
else
{
//back to savedIndex
visitor.SetCurrentIndex(savedIndex);
return(savedIndex);
}
}
else
{
visitor.State = VisitorState.OutOfRangeChar;
return(newBreakAt);
}
}
visitor.SetCurrentIndex(savedIndex);
}
return(0);
}
