public void Run()
{
if (Parameters.Count != 2)
{
throw new ArgumentException("the number of parameters must be 2. Parameter 1 is Analyzer name, Parameter 2 is a text for test.");
}
Analysis.IAnalyzer analyzer = Data.DBProvider.GetAnalyzer(Parameters[0]);
if (analyzer == null)
{
throw new Data.DataException(string.Format("Can't find analyzer name : {0}", Parameters[0]));
}
AddColumn("Word");
AddColumn("Position");
AddColumn("Rank");
foreach (Entity.WordInfo word in analyzer.Tokenize(Parameters[1]))
{
NewRow();
OutputValue("Word", word.Word);
OutputValue("Position", word.Position.ToString());
OutputValue("Rank", word.Rank.ToString());
}
}
/// <summary>
/// Index a text for one field
/// </summary>
/// <param name="text">text</param>
/// <param name="documentId">document id</param>
/// <param name="analyzer">analyzer</param>
private void Index(string text, int documentId, Analysis.IAnalyzer analyzer)
{
lock (this)
{
if (_WordTableWriter == null)
{
_WordTableWriter = new Dictionary <string, int>(65536);
}
if (_DocPositionAlloc == null)
{
_DocPositionAlloc = new DocumentPositionAlloc();
}
_DocumentCount++;
if (_TempWordIndexWriter == null)
{
_TempWordIndexWriter = new AppendList <int>(65536);
}
_TempWordIndexWriter.Clear();
foreach (Entity.WordInfo wordInfo in analyzer.Tokenize(text))
{
if (wordInfo.Position < 0)
{
continue;
}
string internedWord = string.IsInterned(wordInfo.Word);
if (internedWord == null)
{
internedWord = wordInfo.Word;
}
int index;
if (!_WordTableWriter.TryGetValue(internedWord, out index))
{
if (_WordIndexWriterPool == null)
{
_WordIndexWriterPool = new WordIndexWriter[65536];
}
if (_IndexWriterPoolId >= _WordIndexWriterPool.Length)
{
int nextLength = _WordIndexWriterPool.Length * 2;
WordIndexWriter[] tempPool = new WordIndexWriter[nextLength];
Array.Copy(_WordIndexWriterPool, tempPool, _WordIndexWriterPool.Length);
_WordIndexWriterPool = tempPool;
}
_WordIndexWriterPool[_IndexWriterPoolId] = new WordIndexWriter(wordInfo.Word, _IndexMode, _DocPositionAlloc);
_WordIndexWriterPool[_IndexWriterPoolId].TempDocId = documentId;
_WordIndexWriterPool[_IndexWriterPoolId].TempWordCountInThisDoc = 0;
_WordIndexWriterPool[_IndexWriterPoolId].TempFirstPosition = wordInfo.Position;
_WordIndexWriterPool[_IndexWriterPoolId].TempTotalWordsInDoc = analyzer.Count;
_WordTableWriter.Add(wordInfo.Word, _IndexWriterPoolId);
_TempWordIndexWriter.Add(_IndexWriterPoolId);
index = _IndexWriterPoolId;
_IndexWriterPoolId++;
}
if (_WordIndexWriterPool[index].TempDocId != documentId)
{
_WordIndexWriterPool[index].TempDocId = documentId;
_WordIndexWriterPool[index].TempWordCountInThisDoc = 1;
_WordIndexWriterPool[index].TempFirstPosition = wordInfo.Position;
_WordIndexWriterPool[index].TempTotalWordsInDoc = analyzer.Count;
_TempWordIndexWriter.Add(index);
}
else
{
if (_WordIndexWriterPool[index].TempFirstPosition > wordInfo.Position)
{
_WordIndexWriterPool[index].TempFirstPosition = wordInfo.Position;
}
_WordIndexWriterPool[index].TempWordCountInThisDoc++;
}
}
foreach (int writeId in _TempWordIndexWriter)
{
_WordIndexWriterPool[writeId].Index();
}
}
}
public void Run()
{
if (Parameters.Count < 3)
{
throw new ArgumentException("Parameter 1 is table name, Parameter 2 is field name, Parameter 3 is a text for test, Parameter 4 is analyzer type(optional)");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]);
if (dbProvider == null)
{
throw new Data.DataException(string.Format("Can't find table name : {0}", Parameters[0]));
}
Data.Field field = dbProvider.GetField(Parameters[1]);
if (field == null)
{
throw new Data.DataException(string.Format("Can't find field name : {0}", Parameters[2]));
}
if (field.AnalyzerName == null)
{
throw new Data.DataException(string.Format("Field: {0} in {1} is not tokenized", Parameters[2],
dbProvider.Table.Name));
}
Analysis.IAnalyzer analyzer = Data.DBProvider.GetAnalyzer(field.AnalyzerName);
if (analyzer == null)
{
throw new Data.DataException(string.Format("Can't find analyzer name : {0}", field.AnalyzerName));
}
bool clientAnalyzer = false;
if (Parameters.Count == 4)
{
if (Parameters[3].Equals("sqlclient", StringComparison.CurrentCultureIgnoreCase))
{
clientAnalyzer = true;
}
}
AddColumn("Word");
AddColumn("Position");
AddColumn("Rank");
if (clientAnalyzer)
{
foreach (Entity.WordInfo word in analyzer.TokenizeForSqlClient(Parameters[2]))
{
NewRow();
OutputValue("Word", word.Word);
OutputValue("Position", word.Position.ToString());
OutputValue("Rank", word.Rank.ToString());
}
}
else
{
foreach (Entity.WordInfo word in analyzer.Tokenize(Parameters[2]))
{
NewRow();
OutputValue("Word", word.Word);
OutputValue("Position", word.Position.ToString());
OutputValue("Rank", word.Rank.ToString());
}
}
}