public void Run()
{
if (Parameters.Count != 2)
{
throw new ArgumentException("the number of parameters must be 2. Parameter 1 is Analyzer name, Parameter 2 is a text for test.");
}
Analysis.IAnalyzer analyzer = Data.DBProvider.GetAnalyzer(Parameters[0]);
if (analyzer == null)
{
throw new Data.DataException(string.Format("Can't find analyzer name : {0}", Parameters[0]));
}
AddColumn("Word");
AddColumn("Position");
AddColumn("Rank");
foreach (Entity.WordInfo word in analyzer.Tokenize(Parameters[1]))
{
NewRow();
OutputValue("Word", word.Word);
OutputValue("Position", word.Position.ToString());
OutputValue("Rank", word.Rank.ToString());
}
}
public Analysis.IAnalyzer GetAnalyzer()
{
if (string.IsNullOrEmpty(AnalyzerName))
{
_Analyzer = new Analysis.SimpleAnalyzer();
}
else
{
_Analyzer = DBProvider.GetAnalyzer(AnalyzerName);
if (_Analyzer == null)
{
throw new DataException(string.Format("Can't find analyzer name : {0}", AnalyzerName));
}
}
return(_Analyzer);
}
internal void Load(Dictionary <string, Type> dictionary)
{
Assembly asm = Assembly.LoadFrom(AssemblyFile);
foreach (Type type in asm.GetTypes())
{
if (type.GetInterface(Interface.FullName) != null)
{
Data.INamedExternalReference refer = asm.CreateInstance(type.FullName) as
Data.INamedExternalReference;
if (refer == null)
{
Report.WriteErrorLog(string.Format("External reference {0} does not inherit from INamedExternalReference",
type.FullName));
}
else
{
string key = refer.Name.ToLower().Trim();
if (!dictionary.ContainsKey(key))
{
if (refer is Analysis.IAnalyzer)
{
Analysis.IAnalyzer analyzer = refer as Analysis.IAnalyzer;
analyzer.Init();
}
dictionary.Add(key, type);
}
else
{
Global.Report.WriteErrorLog(string.Format("Reduplicate name = {0} in External reference {1}!",
refer.Name, type.FullName));
}
}
}
}
}
public static void Init()
{
#if HubblePro
System.Reflection.Assembly bigTableAsm;
bigTableAsm = System.Reflection.Assembly.LoadFrom("HubblePro.dll");
BigTableParseType = bigTableAsm.GetType("HubblePro.BigTable.BigTableParse");
#else
BigTableParseType = typeof(BigTable.BigTableParse);
#endif
//Init XML Cache File Serialization
Hubble.Framework.Serialization.XmlSerialization <Cache.CacheFile> .Serialize(
new Cache.CacheFile("",
new Hubble.Core.Cache.QueryCacheDocuments(),
new Hubble.Core.Cache.QueryCacheInformation()),
Encoding.UTF8, new System.IO.MemoryStream());
//Load user right from right.db
try
{
Global.UserRightProvider.Load(Global.Setting.UserRigthFilePath);
}
catch (Exception e)
{
Global.Report.WriteErrorLog("Load user right db fail!", e);
}
//Set Ram Index Cache Max Memory
try
{
Hubble.Framework.IO.CachedFileBufferManager.Init(Global.Setting.Config.RamIndexMemoryLimited,
ErrorMessageForCachedFileBufferManager);
}
catch (Exception e)
{
Global.Report.WriteErrorLog("Set RamIndexMemoryLimited fail!", e);
}
//Build QueryTable
System.Reflection.Assembly asm = System.Reflection.Assembly.GetExecutingAssembly();
foreach (Type type in asm.GetTypes())
{
//Init _QueryTable
if (type.GetInterface("Hubble.Core.Query.IQuery") != null)
{
Hubble.Core.Query.IQuery iQuery = asm.CreateInstance(type.FullName) as
Hubble.Core.Query.IQuery;
string key = iQuery.Command.ToLower().Trim();
if (!_QueryTable.ContainsKey(key))
{
_QueryTable.Add(key, type);
}
else
{
Global.Report.WriteErrorLog(string.Format("Reduplicate query command name = {0}",
iQuery.Command));
}
}
//Init _AnalyzerTable
if (type.GetInterface("Hubble.Core.Analysis.IAnalyzer") != null)
{
INamedExternalReference refer = asm.CreateInstance(type.FullName) as
INamedExternalReference;
if (refer == null)
{
Report.WriteErrorLog(string.Format("External reference {0} does not inherit from INamedExternalReference",
type.FullName));
}
else
{
string key = refer.Name.ToLower().Trim();
if (!_AnalyzerTable.ContainsKey(key))
{
Analysis.IAnalyzer analyzer = refer as Analysis.IAnalyzer;
analyzer.Init();
_AnalyzerTable.Add(key, type);
}
else
{
Global.Report.WriteErrorLog(string.Format("Reduplicate name = {0} in External reference {1}!",
refer.Name, type.FullName));
}
}
}
//Init _DBAdapterTable
if (type.GetInterface("Hubble.Core.DBAdapter.IDBAdapter") != null)
{
INamedExternalReference refer = asm.CreateInstance(type.FullName) as
INamedExternalReference;
if (refer == null)
{
Report.WriteErrorLog(string.Format("External reference {0} does not inherit from INamedExternalReference",
type.FullName));
}
else
{
string key = refer.Name.ToLower().Trim();
if (!_DBAdapterTable.ContainsKey(key))
{
_DBAdapterTable.Add(key, type);
}
else
{
Global.Report.WriteErrorLog(string.Format("Reduplicate name = {0} in External reference {1}!",
refer.Name, type.FullName));
}
}
}
if (type.GetInterface("Hubble.Core.StoredProcedure.IStoredProc") != null)
{
Hubble.Core.StoredProcedure.IStoredProc iSP = asm.CreateInstance(type.FullName) as
Hubble.Core.StoredProcedure.IStoredProc;
string key = iSP.Name.ToLower().Trim();
if (!_StoredProcTable.ContainsKey(key))
{
_StoredProcTable.Add(key, type);
}
else
{
Global.Report.WriteErrorLog(string.Format("Reduplicate StoredProcedure name = {0}",
iSP.Name));
}
}
}
//Load from external reference
//Load IQuery external reference
foreach (IQueryConfig iquery in Setting.Config.IQuerys)
{
try
{
if (iquery != null)
{
iquery.Load(_QueryTable);
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("Load IQuery fail. IQuery asm file:{0}", iquery.AssemblyFile),
e);
}
}
//Load IAnalyzer external reference
foreach (IAnalyzerConfig ianalyzer in Setting.Config.IAnalyzers)
{
try
{
if (ianalyzer != null)
{
ianalyzer.Load(_AnalyzerTable);
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("Load IAnalyzer fail. IAnalyzer asm file:{0}", ianalyzer.AssemblyFile),
e);
}
}
//Load IDBAdapter external reference
foreach (IDBAdapterConfig idbadapter in Setting.Config.IDBAdapters)
{
try
{
if (idbadapter != null)
{
idbadapter.Load(_DBAdapterTable);
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("Load IDBAdapter fail. IDBAdapter asm file:{0}", idbadapter.AssemblyFile),
e);
}
}
//Load IDistinct external reference
foreach (IDistinctConfig idistinct in Setting.Config.IDistincts)
{
try
{
if (idistinct != null)
{
idistinct.Load(Core.SFQL.Parse.DistinctInterfaceLoader.ExternalDistincts);
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("Load IDistinct fail. IDistinct asm file:{0}", idistinct.AssemblyFile),
e);
}
}
//List<string> removeDirs = new List<string>();
//Init table
foreach (TableConfig tc in Setting.Config.Tables)
{
try
{
if (tc.Directory == null)
{
continue;
}
DBProvider dbProvider = InitTable(tc);
if (_DBAdapterTable.ContainsKey(dbProvider.TableName.ToLower().Trim()))
{
Global.Report.WriteErrorLog(string.Format("Init Table {0} fail, table exists!",
dbProvider.TableName));
}
else
{
_DBProviderTable.Add(dbProvider.TableName.ToLower().Trim(), dbProvider);
}
}
catch (Exception e)
{
//removeDirs.Add(tc.Directory);
Global.Report.WriteErrorLog(string.Format("Init Table at {0} fail, errmsg:{1} stack {2}",
tc.Directory, e.Message, e.StackTrace));
}
}
Cache.QueryCacheManager.Manager.MaxMemorySize = Global.Setting.Config.QueryCacheMemoryLimited;
//foreach (string removeDir in removeDirs)
//{
// Setting.RemoveTableConfig(removeDir);
//}
}
/// <summary>
/// Index a text for one field
/// </summary>
/// <param name="text">text</param>
/// <param name="documentId">document id</param>
/// <param name="analyzer">analyzer</param>
private void Index(string text, int documentId, Analysis.IAnalyzer analyzer)
{
lock (this)
{
if (_WordTableWriter == null)
{
_WordTableWriter = new Dictionary <string, int>(65536);
}
if (_DocPositionAlloc == null)
{
_DocPositionAlloc = new DocumentPositionAlloc();
}
_DocumentCount++;
if (_TempWordIndexWriter == null)
{
_TempWordIndexWriter = new AppendList <int>(65536);
}
_TempWordIndexWriter.Clear();
foreach (Entity.WordInfo wordInfo in analyzer.Tokenize(text))
{
if (wordInfo.Position < 0)
{
continue;
}
string internedWord = string.IsInterned(wordInfo.Word);
if (internedWord == null)
{
internedWord = wordInfo.Word;
}
int index;
if (!_WordTableWriter.TryGetValue(internedWord, out index))
{
if (_WordIndexWriterPool == null)
{
_WordIndexWriterPool = new WordIndexWriter[65536];
}
if (_IndexWriterPoolId >= _WordIndexWriterPool.Length)
{
int nextLength = _WordIndexWriterPool.Length * 2;
WordIndexWriter[] tempPool = new WordIndexWriter[nextLength];
Array.Copy(_WordIndexWriterPool, tempPool, _WordIndexWriterPool.Length);
_WordIndexWriterPool = tempPool;
}
_WordIndexWriterPool[_IndexWriterPoolId] = new WordIndexWriter(wordInfo.Word, _IndexMode, _DocPositionAlloc);
_WordIndexWriterPool[_IndexWriterPoolId].TempDocId = documentId;
_WordIndexWriterPool[_IndexWriterPoolId].TempWordCountInThisDoc = 0;
_WordIndexWriterPool[_IndexWriterPoolId].TempFirstPosition = wordInfo.Position;
_WordIndexWriterPool[_IndexWriterPoolId].TempTotalWordsInDoc = analyzer.Count;
_WordTableWriter.Add(wordInfo.Word, _IndexWriterPoolId);
_TempWordIndexWriter.Add(_IndexWriterPoolId);
index = _IndexWriterPoolId;
_IndexWriterPoolId++;
}
if (_WordIndexWriterPool[index].TempDocId != documentId)
{
_WordIndexWriterPool[index].TempDocId = documentId;
_WordIndexWriterPool[index].TempWordCountInThisDoc = 1;
_WordIndexWriterPool[index].TempFirstPosition = wordInfo.Position;
_WordIndexWriterPool[index].TempTotalWordsInDoc = analyzer.Count;
_TempWordIndexWriter.Add(index);
}
else
{
if (_WordIndexWriterPool[index].TempFirstPosition > wordInfo.Position)
{
_WordIndexWriterPool[index].TempFirstPosition = wordInfo.Position;
}
_WordIndexWriterPool[index].TempWordCountInThisDoc++;
}
}
foreach (int writeId in _TempWordIndexWriter)
{
_WordIndexWriterPool[writeId].Index();
}
}
}
public void Run()
{
if (Parameters.Count < 3)
{
throw new ArgumentException("Parameter 1 is table name, Parameter 2 is field name, Parameter 3 is a text for test, Parameter 4 is analyzer type(optional)");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]);
if (dbProvider == null)
{
throw new Data.DataException(string.Format("Can't find table name : {0}", Parameters[0]));
}
Data.Field field = dbProvider.GetField(Parameters[1]);
if (field == null)
{
throw new Data.DataException(string.Format("Can't find field name : {0}", Parameters[2]));
}
if (field.AnalyzerName == null)
{
throw new Data.DataException(string.Format("Field: {0} in {1} is not tokenized", Parameters[2],
dbProvider.Table.Name));
}
Analysis.IAnalyzer analyzer = Data.DBProvider.GetAnalyzer(field.AnalyzerName);
if (analyzer == null)
{
throw new Data.DataException(string.Format("Can't find analyzer name : {0}", field.AnalyzerName));
}
bool clientAnalyzer = false;
if (Parameters.Count == 4)
{
if (Parameters[3].Equals("sqlclient", StringComparison.CurrentCultureIgnoreCase))
{
clientAnalyzer = true;
}
}
AddColumn("Word");
AddColumn("Position");
AddColumn("Rank");
if (clientAnalyzer)
{
foreach (Entity.WordInfo word in analyzer.TokenizeForSqlClient(Parameters[2]))
{
NewRow();
OutputValue("Word", word.Word);
OutputValue("Position", word.Position.ToString());
OutputValue("Rank", word.Rank.ToString());
}
}
else
{
foreach (Entity.WordInfo word in analyzer.Tokenize(Parameters[2]))
{
NewRow();
OutputValue("Word", word.Word);
OutputValue("Position", word.Position.ToString());
OutputValue("Rank", word.Rank.ToString());
}
}
}
