unsafe private void TestGetDocIdReplaceFieldValue(string tableName)
{
OutputMessage("TestGetDocIdReplaceFieldValue");
AddColumn("Times");
AddColumn("Elapse(ms)");
AddColumn("ElapseOneTime(ms)");
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0], false);
if (dbProvider == null)
{
throw new DataException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
Random rand = new Random();
int count = 1000000;
int lastDocId = dbProvider.LastDocId;
Stopwatch sw = new Stopwatch();
//int docid = rand.Next(lastDocId);
//int data;
sw.Start();
sw.Stop();
OutputValue("Times", count);
OutputValue("Elapse(ms)", sw.ElapsedMilliseconds);
OutputValue("ElapseOneTime(ms)", (double)sw.ElapsedMilliseconds / count);
}
public void Run()
{
Global.UserRightProvider.CanDo("", Right.RightItem.CreateTable);
if (Parameters.Count != 2)
{
throw new ArgumentException("the number of parameters must be 3. Parameter 1 is table name, parameter 2 is xml string of bigtable.");
}
string tableName = Parameters[0];
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", tableName));
}
System.IO.Stream stream = Hubble.Framework.IO.Stream.WriteStringToStream(Parameters[1], Encoding.UTF8);
BigTable.BigTable bigtable = Hubble.Framework.Serialization.XmlSerialization <BigTable.BigTable> .Deserialize(stream);
bigtable.TimeStamp = DateTime.Now.ToUniversalTime();
dbProvider.BigTableCfg = bigtable;
dbProvider.Table.Save(dbProvider.Directory);
OutputMessage(string.Format("Set bigtable {0} successul.", tableName));
}
void SetValue(string tableName, string value, string timeout)
{
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist.", tableName));
}
bool cacheEnabled;
int cacheTimeout = -1;
if (!bool.TryParse(value, out cacheEnabled))
{
throw new StoredProcException("Parameter 2 must be 'True' or 'False'");
}
if (timeout != null)
{
if (!int.TryParse(timeout, out cacheTimeout))
{
throw new StoredProcException("Parameter 3 must be none-negative integer");
}
}
dbProvider.SetCacheQuery(cacheEnabled, cacheTimeout);
dbProvider.SaveTable();
OutputMessage(string.Format("Table:{0} QueryCacheEnabled = {1} QueryCatchTimeout = {2}",
tableName, dbProvider.QueryCacheEnabled, dbProvider.QueryCacheTimeout));
}
public void Run()
{
Global.UserRightProvider.CanDo("", Right.RightItem.CreateTable);
if (Parameters.Count != 1)
{
throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name");
}
string tableName = Parameters[0];
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
AddColumn("IndexFolder");
AddColumn("BigTable");
NewRow();
OutputValue("IndexFolder", dbProvider.Directory);
string xml;
Hubble.Framework.IO.Stream.ReadStreamToString(
Hubble.Framework.Serialization.XmlSerialization <Hubble.Core.BigTable.BigTable> .Serialize(dbProvider.BigTableCfg),
out xml, Encoding.UTF8);
OutputValue("BigTable", xml);
}
public void Run()
{
if (Parameters.Count != 1)
{
throw new StoredProcException("First parameter is table name.");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
AddColumn("FieldName");
AddColumn("Rate");
foreach (Index.InvertedIndex index in dbProvider.GetInvertedIndexes())
{
NewRow();
OutputValue("FieldName", index.FieldName);
OutputValue("Rate", index.MergeRate);
}
}
public void Run()
{
Global.UserRightProvider.CanDo(Right.RightItem.QuerySql);
if (Parameters.Count < 1)
{
throw new ArgumentException("the number of parameters must be 2. Parameter 1 is table name. Parameter 2 is sql for excute");
}
string sql;
DBAdapter.IDBAdapter dbAdapter;
if (Parameters.Count == 1)
{
string curDatabaseName = Service.CurrentConnection.ConnectionInfo.DatabaseName;
Global.Database database = Global.Setting.GetDatabase(curDatabaseName);
if (string.IsNullOrEmpty(database.DefaultDBAdapter))
{
throw new StoredProcException("Current database hasn't default dbadapter");
}
if (string.IsNullOrEmpty(database.DefaultConnectionString))
{
throw new StoredProcException("Current database hasn't default connectionstring");
}
dbAdapter = (DBAdapter.IDBAdapter)Hubble.Framework.Reflection.Instance.CreateInstance(
Data.DBProvider.GetDBAdapter(database.DefaultDBAdapter));
if (dbAdapter == null)
{
throw new StoredProcException(string.Format("Current database include a invalid default dbadapter:{0}",
database.DefaultDBAdapter));
}
dbAdapter.ConnectionString = database.DefaultConnectionString;
sql = Parameters[0];
}
else
{
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
dbAdapter = dbProvider.DBAdapter;
sql = Parameters[1];
}
_QueryResult.DataSet = new Hubble.Framework.Data.DataSet(dbAdapter.QuerySql(sql));
}
internal InvertedIndex(string path, string fieldName,
Data.Field.IndexMode mode, bool rebuild, Data.DBProvider dbProvider, int documnetCount)
{
_FieldName = fieldName;
_IndexMode = mode;
_DBProvider = dbProvider;
InitFileStore(path, fieldName, rebuild);
//Cache.CacheManager.Register(this);
_IndexMerge = new IndexMerge(path, _IndexFileProxy);
_DocumentCount = documnetCount;
//InitCollectThread();
}
unsafe public static void Generate(ref Docid2Long docid2Long,
Data.DBProvider dbProvider, Data.Field[] orderByFields, long score)
{
int *p = dbProvider.GetPayloadDataWithShareLock(docid2Long.DocId);
if (p == null)
{
throw new Data.DataException(string.Format("DocId={0} does not exist in Payload",
docid2Long.DocId));
}
for (int i = 0; i < orderByFields.Length; i++)
{
Data.Field field = orderByFields[i];
long value;
if (field.Name == "score")
{
value = score;
}
else if (field.Name == "docid")
{
value = docid2Long.DocId;
}
else
{
value = Data.DataTypeConvert.GetLongAnyWay(field.DataType, p,
field.TabIndex, field.SubTabIndex, field.DataLength);
}
switch (i)
{
case 0:
docid2Long.Value1 = value;
break;
case 1:
docid2Long.Value2 = value;
break;
case 2:
docid2Long.Value3 = value;
break;
default:
throw new Data.DataException("Order by fields length for optimization must be <= 3");
}
}
}
void ShowValue(string tableName)
{
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist.", tableName));
}
AddColumn("TableName");
AddColumn("IndexOnly");
OutputValue("TableName", tableName);
OutputValue("IndexOnly", dbProvider.IndexOnly.ToString());
}
//internal void WordUpdate(string word, List<DocumentPositionList> docList)
//{
// lock (this)
// {
// _ListForReader.AddRange(docList);
// UpdateDocScoreList();
// }
//}
public WordIndexReader(string word, WordDocumentsList docList, int totalDocs,
Data.DBProvider dbProvider)
{
_Word = word;
_ListForReader = docList;
_DBProvider = dbProvider;
_DelProvider = _DBProvider.DelProvider;
//_TabIndex = tabIndex;
_TotalDocs = totalDocs;
_WordCountSum = _ListForReader.WordCountSum;
_RelDocCount = _ListForReader.RelDocCount;
_Count = _ListForReader.RelDocCount;
//UpdateDocScoreList();
}
public void Run()
{
Global.UserRightProvider.CanDo(Right.RightItem.Optimize);
if (Parameters.Count > 2 || Parameters.Count <= 0)
{
throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name.");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]);
if (dbProvider == null)
{
OutputMessage(string.Format("Table name {0} does not exist!", Parameters[0]));
return;
}
else
{
int option = 1;
if (Parameters.Count == 2)
{
option = int.Parse(Parameters[1]);
}
switch (option)
{
case 1:
dbProvider.Optimize(Hubble.Core.Data.OptimizationOption.Minimum);
break;
case 2:
dbProvider.Optimize(Hubble.Core.Data.OptimizationOption.Middle);
break;
case 3:
dbProvider.Optimize(Hubble.Core.Data.OptimizationOption.Speedy);
break;
default:
dbProvider.Optimize();
break;
}
}
OutputMessage(string.Format("System optimizing {0} in background now. Maybe need a few minutes to finish it!",
dbProvider.TableName));
}
unsafe private void TestFillPayloadRank(string tableName)
{
OutputMessage("TestGetDocIdReplaceFieldValue");
AddColumn("Times");
AddColumn("Elapse(ms)");
AddColumn("ElapseOneTime(ms)");
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0], false);
if (dbProvider == null)
{
throw new DataException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
Random rand = new Random();
int count = 1000000;
int lastDocId = dbProvider.LastDocId;
OriginalDocumentPositionList[] payloads = new OriginalDocumentPositionList[count];
for (int i = 0; i < payloads.Length; i++)
{
payloads[i] = new OriginalDocumentPositionList(i * 10);
payloads[i].CountAndWordCount = 1;
}
payloads[0].DocumentId = 8 * payloads.Length;
Data.Field rankField = dbProvider.GetField("Rank");
int tab = rankField.TabIndex;
Stopwatch sw = new Stopwatch();
//int docid = rand.Next(lastDocId);
sw.Start();
for (int j = 0; j < count / payloads.Length; j++)
{
dbProvider.FillPayloadRank(tab, count, payloads);
}
sw.Stop();
OutputValue("Times", count);
OutputValue("Elapse(ms)", sw.ElapsedMilliseconds);
OutputValue("ElapseOneTime(ms)", (double)sw.ElapsedMilliseconds / count);
}
void ShowValue(string tableName)
{
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist.", tableName));
}
AddColumn("TableName");
AddColumn("QueryCacheEnabled");
AddColumn("QueryCacheTimeout");
OutputValue("TableName", tableName);
OutputValue("QueryCacheEnabled", dbProvider.QueryCacheEnabled.ToString());
OutputValue("QueryCacheTimeout", dbProvider.QueryCacheTimeout.ToString());
}
public void Run()
{
Global.UserRightProvider.CanDo(Right.RightItem.Select);
if (Parameters.Count < 2)
{
throw new ArgumentException("Parameter 1 is select statement before where. Parameter 2 - n is docids.");
}
string sql = Parameters[0];
SFQL.Parse.SFQLParse sfqlParse = new Hubble.Core.SFQL.Parse.SFQLParse();
sfqlParse.SyntaxAnalyse(sql);
SFQL.SyntaxAnalysis.Select.Select select = sfqlParse.SFQLSentenceList[0].SyntaxEntity as
SFQL.SyntaxAnalysis.Select.Select;
string tableName = select.SelectFroms[0].Name;
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new Data.DataException(string.Format("Invalid table:{0}", tableName));
}
List <Data.Field> fields;
int allFieldsCount;
sfqlParse.GetSelectFields(select, dbProvider, out fields, out allFieldsCount);
Query.DocumentResultForSort[] docs = new Hubble.Core.Query.DocumentResultForSort[Parameters.Count - 1];
for (int i = 1; i < Parameters.Count; i++)
{
docs[i - 1] = new Hubble.Core.Query.DocumentResultForSort(int.Parse(Parameters[i]));
}
List <Data.Document> docResult = dbProvider.Query(fields, docs);
Hubble.Framework.Data.DataSet ds = Data.Document.ToDataSet(fields, docResult);
ds.Tables[0].TableName = "Select_" + select.SelectFroms[0].Alias;
_QueryResult.DataSet = ds;
}
public void Run()
{
Global.UserRightProvider.CanDo(Right.RightItem.ManageDB);
if (Parameters.Count != 1)
{
throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
dbProvider.StopSynchronize();
OutputMessage(string.Format("Table: {0}'s synchronization is stopping now!", Parameters[0]));
}
public void Run()
{
if (Parameters.Count != 1)
{
throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name.");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0], false);
if (dbProvider == null)
{
throw new Data.DataException(string.Format("Table name {0} does not exist!", Parameters[0]));
//OutputMessage(string.Format("Table name {0} does not exist!", Parameters[0]));
}
else
{
AddColumn("FieldName");
AddColumn("DataType");
AddColumn("DataLength");
AddColumn("IndexType");
AddColumn("Analyzer");
AddColumn("IsNull");
AddColumn("IsPrimaryKey");
AddColumn("Default");
foreach (Data.Field field in dbProvider.GetAllFields())
{
NewRow();
OutputValue("FieldName", field.Name);
OutputValue("DataType", field.DataType.ToString());
OutputValue("DataLength", field.DataLength);
OutputValue("IndexType", field.IndexType.ToString());
OutputValue("Analyzer", field.AnalyzerName);
OutputValue("IsNull", field.CanNull);
OutputValue("IsPrimaryKey", field.PrimaryKey);
OutputValue("Default", field.DefaultValue);
}
}
}
public void Run()
{
Global.UserRightProvider.CanDo(Right.RightItem.DropTable);
if (Parameters.Count != 1)
{
throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name.");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0], false);
if (dbProvider == null)
{
OutputMessage(string.Format("Table name {0} does not exist!", Parameters[0]));
}
else
{
Data.DBProvider.Drop(Parameters[0]);
}
OutputMessage(string.Format("Drop table {0} successul.", Parameters[0]));
}
public void Run()
{
Global.UserRightProvider.CanDo(Right.RightItem.ManageDB);
if (Parameters.Count != 1)
{
throw new ArgumentException("the number of parameters must be 1. Parameter 1 is tableName");
}
string tableName = Parameters[0];
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
dbProvider.Detach();
OutputMessage(string.Format("Detach table {0} successul.", tableName));
}
public void Run()
{
if (Parameters.Count != 1)
{
throw new StoredProcException("First parameter is table name.");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
AddColumn("Progress");
AddColumn("InsertRows");
NewRow();
OutputValue("Progress", dbProvider.TableSynchronizeProgress);
OutputValue("InsertRows", dbProvider.TableSynchronizeInsertRows);
}
void SetValue(string tableName, string value)
{
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist.", tableName));
}
bool indexonly;
if (bool.TryParse(value, out indexonly))
{
dbProvider.SetIndexOnly(indexonly);
dbProvider.SaveTable();
OutputMessage(string.Format("Set table {0} index only to {1} sucessful!",
tableName, dbProvider.IndexOnly));
}
else
{
throw new StoredProcException("Parameter 2 must be 'True' or 'False'");
}
}
public static DocId2LongComparer Generate(Data.DBProvider dbProvider,
OrderBy[] orderBys, out Data.Field[] orderByFields)
{
bool[] ascs = new bool[orderBys.Length];
orderByFields = new Hubble.Core.Data.Field[orderBys.Length];
int scoreFieldIndex = -1;
for (int i = 0; i < ascs.Length; i++)
{
if (orderBys[i].Order == null)
{
ascs[i] = true;
}
else
{
ascs[i] = !orderBys[i].Order.Equals("desc", StringComparison.CurrentCultureIgnoreCase);
}
if (orderBys[i].Name.Equals("docid", StringComparison.CurrentCultureIgnoreCase))
{
orderByFields[i] = new Hubble.Core.Data.Field("docid", Hubble.Core.Data.DataType.Int);
}
else if (orderBys[i].Name.Equals("score", StringComparison.CurrentCultureIgnoreCase))
{
scoreFieldIndex = i;
orderByFields[i] = new Hubble.Core.Data.Field("score", Hubble.Core.Data.DataType.BigInt);
}
else
{
orderByFields[i] = dbProvider.GetField(orderBys[i].Name);
}
}
return(new DocId2LongComparer(ascs, scoreFieldIndex));
}
public WordIndexReader(string word, WordStepDocIndex wordStepDocIndex, int totalDocs,
Data.DBProvider dbProvider, IndexFileProxy indexProxy, int maxReturnCount)
{
_Word = word;
_IndexReader = new IndexReader(wordStepDocIndex, indexProxy);
_DBProvider = dbProvider;
_DelProvider = _DBProvider.DelProvider;
//_TabIndex = tabIndex;
_TotalDocs = totalDocs;
_WordCountSum = wordStepDocIndex.WordCountSum;
_RelDocCount = wordStepDocIndex.RelDocCount;
if (maxReturnCount < 0)
{
_Count = _RelDocCount;
}
else
{
_Count = Math.Min(maxReturnCount, _RelDocCount);
}
//UpdateDocScoreList();
}
public void Run()
{
if (Parameters.Count != 1)
{
throw new StoredProcException("First parameter is table name.");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
AddColumn("Attribute");
AddColumn("Value");
NewRow();
OutputValue("Attribute", "Directory");
OutputValue("Value", dbProvider.Directory);
NewRow();
OutputValue("Attribute", "IndexOnly");
OutputValue("Value", dbProvider.IndexOnly.ToString());
NewRow();
OutputValue("Attribute", "DocId");
OutputValue("Value", dbProvider.DocIdReplaceField);
NewRow();
OutputValue("Attribute", "DBTableName");
OutputValue("Value", dbProvider.Table.DBTableName);
NewRow();
OutputValue("Attribute", "DBAdapter");
OutputValue("Value", ((Data.INamedExternalReference)dbProvider.DBAdapter).Name);
NewRow();
OutputValue("Attribute", "LastDocId");
OutputValue("Value", dbProvider.LastDocIdForIndexOnly.ToString());
NewRow();
OutputValue("Attribute", "MaxReturnCount");
OutputValue("Value", dbProvider.MaxReturnCount.ToString());
NewRow();
OutputValue("Attribute", "GroupByLimit");
OutputValue("Value", dbProvider.Table.GroupByLimit.ToString());
NewRow();
OutputValue("Attribute", "InitImmediatelyAfterStartup");
OutputValue("Value", dbProvider.Table.InitImmediatelyAfterStartup.ToString());
NewRow();
OutputValue("Attribute", "RamIndexType");
OutputValue("Value", dbProvider.Table.RamIndexType.ToString());
NewRow();
OutputValue("Attribute", "RamIndexMinLoadSize");
OutputValue("Value", dbProvider.Table.RamIndexMinLoadSize.ToString());
NewRow();
OutputValue("Attribute", "QueryCacheEnabled");
OutputValue("Value", dbProvider.QueryCacheEnabled.ToString());
NewRow();
OutputValue("Attribute", "QueryCacheTimeout");
OutputValue("Value", dbProvider.QueryCacheTimeout.ToString());
NewRow();
OutputValue("Attribute", "StoreQueryCacheInFile");
OutputValue("Value", dbProvider.Table.StoreQueryCacheInFile.ToString());
NewRow();
OutputValue("Attribute", "CleanupQueryCacheFileInDays");
OutputValue("Value", dbProvider.Table.CleanupQueryCacheFileInDays.ToString());
NewRow();
OutputValue("Attribute", "IndexThread");
OutputValue("Value", dbProvider.Table.IndexThread.ToString());
NewRow();
OutputValue("Attribute", "Debug");
OutputValue("Value", dbProvider.Table.Debug.ToString());
NewRow();
OutputValue("Attribute", "TableSynchronization");
OutputValue("Value", dbProvider.Table.TableSynchronization.ToString());
NewRow();
OutputValue("Attribute", "TriggerTableName");
OutputValue("Value", dbProvider.Table.TriggerTableName);
NewRow();
OutputValue("Attribute", "MirrorTableEnabled");
OutputValue("Value", dbProvider.Table.MirrorTableEnabled);
NewRow();
OutputValue("Attribute", "UsingMirrorTableForNonFulltextQuery");
OutputValue("Value", dbProvider.Table.UsingMirrorTableForNonFulltextQuery);
NewRow();
OutputValue("Attribute", "SelectTimeout");
OutputValue("Value", dbProvider.Table.SelectTimeout);
}
public void Run()
{
Global.UserRightProvider.CanDo(Right.RightItem.ManageDB);
if (Parameters.Count < 1)
{
throw new ArgumentException("the number of parameters must large than 0. Parameter 1 is table name, Parameter 2 is step, Parameter 3 is optimize mode");
}
string tableName = Parameters[0];
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
if (dbProvider.TableSynchronizeProgress >= 0 &&
dbProvider.TableSynchronizeProgress < 100)
{
return;
}
int option = 0;
int step = 5000;
bool fastestMode = true;
string strFlags = "";
Service.SyncFlags flags = Service.SyncFlags.Insert | Service.SyncFlags.Rebuild;
if (Parameters.Count > 1)
{
step = int.Parse(Parameters[1]);
}
if (Parameters.Count > 2)
{
option = int.Parse(Parameters[2]);
}
if (Parameters.Count > 3)
{
strFlags = Parameters[3];
flags = (Service.SyncFlags)Hubble.Framework.Reflection.Emun.FromString(typeof(Service.SyncFlags), strFlags);
flags &= ~Hubble.Core.Service.SyncFlags.Delete;
flags &= ~Hubble.Core.Service.SyncFlags.Update;
flags |= Hubble.Core.Service.SyncFlags.Rebuild;
flags |= Hubble.Core.Service.SyncFlags.Insert;
}
Hubble.Core.Data.OptimizationOption optimizeOption;
switch (option)
{
case 0:
optimizeOption = Hubble.Core.Data.OptimizationOption.Idle;
break;
case 1:
optimizeOption = Hubble.Core.Data.OptimizationOption.Minimum;
break;
case 2:
optimizeOption = Hubble.Core.Data.OptimizationOption.Middle;
break;
case 3:
optimizeOption = Hubble.Core.Data.OptimizationOption.Speedy;
break;
default:
optimizeOption = Hubble.Core.Data.OptimizationOption.Minimum;
break;
}
bool notIndexOnly = false;
try
{
if (!dbProvider.IndexOnly)
{
notIndexOnly = true;
dbProvider.Table.IndexOnly = true;
}
Data.DBProvider.Truncate(tableName);
if (notIndexOnly)
{
dbProvider = Data.DBProvider.GetDBProvider(tableName);
dbProvider.Table.IndexOnly = false;
}
}
catch
{
if (notIndexOnly)
{
dbProvider = Data.DBProvider.GetDBProvider(tableName);
dbProvider.Table.IndexOnly = false;
}
}
dbProvider = Data.DBProvider.GetDBProvider(tableName);
dbProvider.SynchronizeWithDatabase(step, optimizeOption, fastestMode, flags);
if ((flags & Hubble.Core.Service.SyncFlags.WaitForExit) != 0)
{
//Wait for exit
while (true)
{
if (dbProvider.TableSynchronizeProgress >= 100 ||
dbProvider.TableSynchronizeProgress < 0)
{
OutputMessage(string.Format("Table: {0} finished synchronizing!", Parameters[0]));
return;
}
System.Threading.Thread.Sleep(1000);
}
}
OutputMessage(string.Format("Table: {0} is synchronizing now!", Parameters[0]));
}
unsafe private string GetInSql(Data.DBProvider dbProvider, Query.DocumentResultForSort[] result, int begin, int count)
{
if (begin + count > result.Length)
{
return(null);
}
StringBuilder sql = new StringBuilder();
if (dbProvider.DocIdReplaceField == null)
{
sql.Append("docId in (");
}
else
{
sql.AppendFormat("{0} in (", dbProvider.DocIdReplaceField);
}
Dictionary <long, int> replaceFieldValueToDocId = null;
if (dbProvider.DocIdReplaceField != null)
{
replaceFieldValueToDocId = new Dictionary <long, int>();
}
int i = 0;
for (int j = begin; j < begin + count; j++)
{
Query.DocumentResultForSort docResult = result[j];
int docId = docResult.DocId;
if (dbProvider.DocIdReplaceField == null)
{
if (i++ == 0)
{
sql.AppendFormat("{0}", docId);
}
else
{
sql.AppendFormat(",{0}", docId);
}
}
else
{
long replaceFieldValue = dbProvider.GetDocIdReplaceFieldValue(docId);
if (i++ == 0)
{
sql.AppendFormat("{0}", replaceFieldValue);
}
else
{
sql.AppendFormat(",{0}", replaceFieldValue);
}
}
}
sql.Append(")");
return(sql.ToString());
}
public void Run()
{
if (Parameters.Count != 3)
{
throw new StoredProcException("First parameter is table name, second parameter is field name, third parameter is words. SP_GetIDF 'tablename', 'fieldname', 'abc news'");
}
string tableName = Parameters[0];
string fieldName = Parameters[1];
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", tableName));
}
Hubble.Core.Index.InvertedIndex invertedIndex = dbProvider.GetInvertedIndex(fieldName);
if (invertedIndex == null)
{
throw new StoredProcException(string.Format("Field name {0} does not exist or is not the tokenized index field!", fieldName));
}
string queryStr = Parameters[2];
List <WordInfo> wordInfos = ParseWhere.GetWordInfoList(queryStr);
Dictionary <string, WordIndexForQuery> wordIndexDict = new Dictionary <string, WordIndexForQuery>();
foreach (Hubble.Core.Entity.WordInfo wordInfo in wordInfos)
{
WordIndexForQuery wifq;
if (!wordIndexDict.TryGetValue(wordInfo.Word, out wifq))
{
//Hubble.Core.Index.WordIndexReader wordIndex = InvertedIndex.GetWordIndex(wordInfo.Word, CanLoadPartOfDocs); //Get whole index
Hubble.Core.Index.WordIndexReader wordIndex = invertedIndex.GetWordIndex(wordInfo.Word, false, true); //Only get step doc index
if (wordIndex == null)
{
wordIndexDict.Add(wordInfo.Word, null);
continue;
}
wifq = new WordIndexForQuery(wordIndex,
invertedIndex.DocumentCount, wordInfo.Rank, 1);
wifq.QueryCount = 1;
wifq.FirstPosition = wordInfo.Position;
wordIndexDict.Add(wordInfo.Word, wifq);
}
else
{
wifq.WordRank += wordInfo.Rank;
wifq.QueryCount++;
}
//wordIndexList[wordIndexList.Count - 1].Rank += wordInfo.Rank;
}
AddColumn("Word");
AddColumn("TF");
AddColumn("IDF");
AddColumn("T_D");
AddColumn("TotalDoucments");
AddColumn("TF_IDF");
int totalDocuments = invertedIndex.DocumentCount;
foreach (string word in wordIndexDict.Keys)
{
NewRow();
WordIndexForQuery wifq = wordIndexDict[word];
OutputValue("Word", word);
if (wifq == null)
{
OutputValue("TF", 0);
OutputValue("T_D", 0);
OutputValue("TotalDoucments", 0);
OutputValue("IDF", 0);
OutputValue("TF_IDF", 0);
}
else
{
double idf = Math.Log((double)totalDocuments / (double)wifq.RelTotalCount);
OutputValue("TF", wifq.QueryCount);
OutputValue("T_D", wifq.RelTotalCount);
OutputValue("TotalDoucments", totalDocuments);
OutputValue("IDF", idf);
OutputValue("TF_IDF", wifq.QueryCount * idf);
}
}
}
public void Run()
{
Global.UserRightProvider.CanDo(Right.RightItem.ManageDB);
if (Parameters.Count < 1)
{
throw new ArgumentException("the number of parameters must large than 0. Parameter 1 is table name, Parameter 2 is step, Parameter 3 is optimize mode");
}
Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]);
if (dbProvider == null)
{
throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0]));
}
int option = 0;
int step = 5000;
bool fastestMode = false;
string strFlags = "";
Service.SyncFlags flags = Service.SyncFlags.Insert | Service.SyncFlags.Delete
| Service.SyncFlags.Update;
if (Parameters.Count > 1)
{
step = int.Parse(Parameters[1]);
}
if (Parameters.Count > 2)
{
option = int.Parse(Parameters[2]);
}
if (Parameters.Count > 3)
{
fastestMode = bool.Parse(Parameters[3]);
}
if (Parameters.Count > 4)
{
strFlags = Parameters[4];
flags = (Service.SyncFlags)Hubble.Framework.Reflection.Emun.FromString(typeof(Service.SyncFlags), strFlags);
}
Hubble.Core.Data.OptimizationOption optimizeOption;
switch (option)
{
case 0:
optimizeOption = Hubble.Core.Data.OptimizationOption.Idle;
break;
case 1:
optimizeOption = Hubble.Core.Data.OptimizationOption.Minimum;
break;
case 2:
optimizeOption = Hubble.Core.Data.OptimizationOption.Middle;
break;
case 3:
optimizeOption = Hubble.Core.Data.OptimizationOption.Speedy;
break;
default:
optimizeOption = Hubble.Core.Data.OptimizationOption.Minimum;
break;
}
dbProvider.SynchronizeWithDatabase(step, optimizeOption, fastestMode, flags);
if ((flags & Hubble.Core.Service.SyncFlags.WaitForExit) != 0)
{
//Wait for exit
int count = 0;
while (true)
{
if (dbProvider.TableSynchronizeProgress >= 100 ||
dbProvider.TableSynchronizeProgress < 0)
{
OutputMessage(string.Format("Table: {0} finished synchronizing!", Parameters[0]));
return;
}
if ((++count % 60) == 0)
{
Hubble.Core.Global.Report.WriteAppLog(string.Format("TableSynchronizeProgress = {0}% of table:{1}",
dbProvider.TableSynchronizeProgress, dbProvider.TableName));
}
System.Threading.Thread.Sleep(1000);
}
}
OutputMessage(string.Format("Table: {0} is synchronizing now!", Parameters[0]));
}
internal Hubble.Core.Index.WordIndexReader GetWordIndex(string word,
WordFilePositionList filePositionList, int totalDocs, Data.DBProvider dbProvider, int maxReturnCount)
{
WordDocumentsList docList = new WordDocumentsList();
bool simple = _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple;
if (maxReturnCount < 0)
{
foreach (FilePosition filePosition in filePositionList.FPList)
{
IDXFile idxFile = GetIDXFile(filePosition.Serial);
if (idxFile == null)
{
continue;
}
WordDocumentsList wdl = idxFile.GetDocList(filePosition.Position, filePosition.Length, -1, simple);
if (filePositionList.Count == 1)
{
docList = wdl;
}
else
{
docList.AddRange(wdl);
docList.WordCountSum += wdl.WordCountSum;
}
}
}
else
{
int remain = maxReturnCount;
foreach (FilePosition filePosition in filePositionList.FPList)
{
IDXFile idxFile = GetIDXFile(filePosition.Serial);
if (idxFile == null)
{
continue;
}
WordDocumentsList wdl = idxFile.GetDocList(filePosition.Position, filePosition.Length, remain, simple);
if (filePositionList.Count == 1)
{
docList = wdl;
}
else
{
docList.AddRange(wdl);
docList.WordCountSum += wdl.WordCountSum;
docList.RelDocCount += wdl.RelDocCount;
}
remain -= wdl.Count;
}
}
return(new WordIndexReader(word, docList, totalDocs, dbProvider));
}
/// <summary>
/// Only get the word step doc index.
/// </summary>
/// <param name="word"></param>
/// <param name="filePositionList"></param>
/// <param name="totalDocs"></param>
/// <param name="dbProvider"></param>
/// <param name="maxReturnCount"></param>
/// <returns></returns>
internal Hubble.Core.Index.WordIndexReader GetWordIndexWithWordStepDocIndex(string word,
WordFilePositionList filePositionList, int totalDocs, Data.DBProvider dbProvider, int maxReturnCount)
{
List <WordStepDocIndex.IndexFileInfo> indexFileInfoList =
new List <WordStepDocIndex.IndexFileInfo>(); //StepDocIndex of all index files
bool simple = _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple;
long wordCountSum = 0;
foreach (FilePosition filePosition in filePositionList.FPList)
{
IDXFile idxFile = GetIDXFile(filePosition.Serial);
if (idxFile == null)
{
continue;
}
int count;
long indexPosition;
List <Hubble.Core.Entity.DocumentPosition> stepDocIndex =
idxFile.GetStepDocIndex(filePosition.Position, filePosition.Length, out count, out indexPosition);
indexFileInfoList.Add(new WordStepDocIndex.IndexFileInfo(stepDocIndex,
new FilePosition(filePosition.Serial, indexPosition,
(int)(filePosition.Length - (indexPosition - filePosition.Position) - sizeof(int))), //Subtract last 4 bytes for last doc id and Subtract stepdocindex length
count));
wordCountSum += count;
}
int relDocCount = (int)wordCountSum;
//if (maxReturnCount >= 0)
//{
// if (wordCountSum > maxReturnCount)
// {
// relDocCount = maxReturnCount;
// }
//}
WordStepDocIndex wordStepDocIndex = new WordStepDocIndex(indexFileInfoList,
wordCountSum, relDocCount, simple);
return(new WordIndexReader(word, wordStepDocIndex, totalDocs,
dbProvider, _IndexFileProxy, maxReturnCount));
}
/// <summary>
/// Get IdFields List from trigger table
/// </summary>
/// <param name="dbProvider">dbProvider that used to get field info</param>
/// <param name="dbAdapterName">DBAdapter name</param>
/// <param name="table">table that read from trigger table</param>
/// <param name="lastSerial">the last serial number of trigger table from which read</param>
/// <returns></returns>
internal static List <IdFields> GetIdFieldsList(Data.DBProvider dbProvider, string dbAdapterName,
System.Data.DataTable table, out long lastSerial)
{
lastSerial = -1;
HashSet <string> fieldsSetWithTokenizedFields = new HashSet <string>();
HashSet <string> fieldsSetWithoutTokenizedFields = new HashSet <string>();
List <string> tempFields = new List <string>(128);
List <IdFields> result = new List <IdFields>(table.Rows.Count);
foreach (System.Data.DataRow row in table.Rows)
{
long id = long.Parse(row["id"].ToString());
lastSerial = long.Parse(row["Serial"].ToString());
string fields = row["Fields"].ToString();
tempFields.Clear();
bool hasTokenized = false;
//check fields
foreach (string field in fields.Split(new char[] { ',' }))
{
string f = field.Trim().ToLower();
if (f == "")
{
continue;
}
Data.Field dbField = dbProvider.GetField(f);
if (dbField == null)
{
continue;
}
if (dbField.IndexType == Hubble.Core.Data.Field.Index.Tokenized)
{
hasTokenized = true;
}
tempFields.Add(f);
}
//Fill hash set
if (hasTokenized)
{
foreach (string field in tempFields)
{
if (!fieldsSetWithTokenizedFields.Contains(field))
{
fieldsSetWithTokenizedFields.Add(field);
}
}
}
else
{
foreach (string field in tempFields)
{
if (!fieldsSetWithoutTokenizedFields.Contains(field))
{
fieldsSetWithoutTokenizedFields.Add(field);
}
}
}
result.Add(new IdFields(id, hasTokenized));
}
//Get new fields string
string fieldsWithTokenized = GetFieldsStringFromHashSet(fieldsSetWithTokenizedFields, dbAdapterName);
string fieldsWithoutTokenized = GetFieldsStringFromHashSet(fieldsSetWithoutTokenizedFields, dbAdapterName);
foreach (IdFields idFields in result)
{
if (idFields.HasTokenizedFields)
{
idFields.Fields = fieldsWithTokenized;
}
else
{
idFields.Fields = fieldsWithoutTokenized;
}
}
//Merge same Id
result.Sort();
if (result.Count > 0)
{
IdFields last = result[0];
for (int i = 1; i < result.Count; i++)
{
if (result[i].Equals(last))
{
result[i] = null;
continue;
}
else
{
last = result[i];
}
}
}
return(result);
}
