unsafe private void TestGetDocIdReplaceFieldValue(string tableName) { OutputMessage("TestGetDocIdReplaceFieldValue"); AddColumn("Times"); AddColumn("Elapse(ms)"); AddColumn("ElapseOneTime(ms)"); Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0], false); if (dbProvider == null) { throw new DataException(string.Format("Table name {0} does not exist!", Parameters[0])); } Random rand = new Random(); int count = 1000000; int lastDocId = dbProvider.LastDocId; Stopwatch sw = new Stopwatch(); //int docid = rand.Next(lastDocId); //int data; sw.Start(); sw.Stop(); OutputValue("Times", count); OutputValue("Elapse(ms)", sw.ElapsedMilliseconds); OutputValue("ElapseOneTime(ms)", (double)sw.ElapsedMilliseconds / count); }
public void Run() { Global.UserRightProvider.CanDo("", Right.RightItem.CreateTable); if (Parameters.Count != 2) { throw new ArgumentException("the number of parameters must be 3. Parameter 1 is table name, parameter 2 is xml string of bigtable."); } string tableName = Parameters[0]; Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", tableName)); } System.IO.Stream stream = Hubble.Framework.IO.Stream.WriteStringToStream(Parameters[1], Encoding.UTF8); BigTable.BigTable bigtable = Hubble.Framework.Serialization.XmlSerialization <BigTable.BigTable> .Deserialize(stream); bigtable.TimeStamp = DateTime.Now.ToUniversalTime(); dbProvider.BigTableCfg = bigtable; dbProvider.Table.Save(dbProvider.Directory); OutputMessage(string.Format("Set bigtable {0} successul.", tableName)); }
void SetValue(string tableName, string value, string timeout) { Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist.", tableName)); } bool cacheEnabled; int cacheTimeout = -1; if (!bool.TryParse(value, out cacheEnabled)) { throw new StoredProcException("Parameter 2 must be 'True' or 'False'"); } if (timeout != null) { if (!int.TryParse(timeout, out cacheTimeout)) { throw new StoredProcException("Parameter 3 must be none-negative integer"); } } dbProvider.SetCacheQuery(cacheEnabled, cacheTimeout); dbProvider.SaveTable(); OutputMessage(string.Format("Table:{0} QueryCacheEnabled = {1} QueryCatchTimeout = {2}", tableName, dbProvider.QueryCacheEnabled, dbProvider.QueryCacheTimeout)); }
public void Run() { Global.UserRightProvider.CanDo("", Right.RightItem.CreateTable); if (Parameters.Count != 1) { throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name"); } string tableName = Parameters[0]; Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0])); } AddColumn("IndexFolder"); AddColumn("BigTable"); NewRow(); OutputValue("IndexFolder", dbProvider.Directory); string xml; Hubble.Framework.IO.Stream.ReadStreamToString( Hubble.Framework.Serialization.XmlSerialization <Hubble.Core.BigTable.BigTable> .Serialize(dbProvider.BigTableCfg), out xml, Encoding.UTF8); OutputValue("BigTable", xml); }
public void Run() { if (Parameters.Count != 1) { throw new StoredProcException("First parameter is table name."); } Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0])); } AddColumn("FieldName"); AddColumn("Rate"); foreach (Index.InvertedIndex index in dbProvider.GetInvertedIndexes()) { NewRow(); OutputValue("FieldName", index.FieldName); OutputValue("Rate", index.MergeRate); } }
public void Run() { Global.UserRightProvider.CanDo(Right.RightItem.QuerySql); if (Parameters.Count < 1) { throw new ArgumentException("the number of parameters must be 2. Parameter 1 is table name. Parameter 2 is sql for excute"); } string sql; DBAdapter.IDBAdapter dbAdapter; if (Parameters.Count == 1) { string curDatabaseName = Service.CurrentConnection.ConnectionInfo.DatabaseName; Global.Database database = Global.Setting.GetDatabase(curDatabaseName); if (string.IsNullOrEmpty(database.DefaultDBAdapter)) { throw new StoredProcException("Current database hasn't default dbadapter"); } if (string.IsNullOrEmpty(database.DefaultConnectionString)) { throw new StoredProcException("Current database hasn't default connectionstring"); } dbAdapter = (DBAdapter.IDBAdapter)Hubble.Framework.Reflection.Instance.CreateInstance( Data.DBProvider.GetDBAdapter(database.DefaultDBAdapter)); if (dbAdapter == null) { throw new StoredProcException(string.Format("Current database include a invalid default dbadapter:{0}", database.DefaultDBAdapter)); } dbAdapter.ConnectionString = database.DefaultConnectionString; sql = Parameters[0]; } else { Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0])); } dbAdapter = dbProvider.DBAdapter; sql = Parameters[1]; } _QueryResult.DataSet = new Hubble.Framework.Data.DataSet(dbAdapter.QuerySql(sql)); }
internal InvertedIndex(string path, string fieldName, Data.Field.IndexMode mode, bool rebuild, Data.DBProvider dbProvider, int documnetCount) { _FieldName = fieldName; _IndexMode = mode; _DBProvider = dbProvider; InitFileStore(path, fieldName, rebuild); //Cache.CacheManager.Register(this); _IndexMerge = new IndexMerge(path, _IndexFileProxy); _DocumentCount = documnetCount; //InitCollectThread(); }
unsafe public static void Generate(ref Docid2Long docid2Long, Data.DBProvider dbProvider, Data.Field[] orderByFields, long score) { int *p = dbProvider.GetPayloadDataWithShareLock(docid2Long.DocId); if (p == null) { throw new Data.DataException(string.Format("DocId={0} does not exist in Payload", docid2Long.DocId)); } for (int i = 0; i < orderByFields.Length; i++) { Data.Field field = orderByFields[i]; long value; if (field.Name == "score") { value = score; } else if (field.Name == "docid") { value = docid2Long.DocId; } else { value = Data.DataTypeConvert.GetLongAnyWay(field.DataType, p, field.TabIndex, field.SubTabIndex, field.DataLength); } switch (i) { case 0: docid2Long.Value1 = value; break; case 1: docid2Long.Value2 = value; break; case 2: docid2Long.Value3 = value; break; default: throw new Data.DataException("Order by fields length for optimization must be <= 3"); } } }
void ShowValue(string tableName) { Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist.", tableName)); } AddColumn("TableName"); AddColumn("IndexOnly"); OutputValue("TableName", tableName); OutputValue("IndexOnly", dbProvider.IndexOnly.ToString()); }
//internal void WordUpdate(string word, List<DocumentPositionList> docList) //{ // lock (this) // { // _ListForReader.AddRange(docList); // UpdateDocScoreList(); // } //} public WordIndexReader(string word, WordDocumentsList docList, int totalDocs, Data.DBProvider dbProvider) { _Word = word; _ListForReader = docList; _DBProvider = dbProvider; _DelProvider = _DBProvider.DelProvider; //_TabIndex = tabIndex; _TotalDocs = totalDocs; _WordCountSum = _ListForReader.WordCountSum; _RelDocCount = _ListForReader.RelDocCount; _Count = _ListForReader.RelDocCount; //UpdateDocScoreList(); }
public void Run() { Global.UserRightProvider.CanDo(Right.RightItem.Optimize); if (Parameters.Count > 2 || Parameters.Count <= 0) { throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name."); } Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]); if (dbProvider == null) { OutputMessage(string.Format("Table name {0} does not exist!", Parameters[0])); return; } else { int option = 1; if (Parameters.Count == 2) { option = int.Parse(Parameters[1]); } switch (option) { case 1: dbProvider.Optimize(Hubble.Core.Data.OptimizationOption.Minimum); break; case 2: dbProvider.Optimize(Hubble.Core.Data.OptimizationOption.Middle); break; case 3: dbProvider.Optimize(Hubble.Core.Data.OptimizationOption.Speedy); break; default: dbProvider.Optimize(); break; } } OutputMessage(string.Format("System optimizing {0} in background now. Maybe need a few minutes to finish it!", dbProvider.TableName)); }
unsafe private void TestFillPayloadRank(string tableName) { OutputMessage("TestGetDocIdReplaceFieldValue"); AddColumn("Times"); AddColumn("Elapse(ms)"); AddColumn("ElapseOneTime(ms)"); Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0], false); if (dbProvider == null) { throw new DataException(string.Format("Table name {0} does not exist!", Parameters[0])); } Random rand = new Random(); int count = 1000000; int lastDocId = dbProvider.LastDocId; OriginalDocumentPositionList[] payloads = new OriginalDocumentPositionList[count]; for (int i = 0; i < payloads.Length; i++) { payloads[i] = new OriginalDocumentPositionList(i * 10); payloads[i].CountAndWordCount = 1; } payloads[0].DocumentId = 8 * payloads.Length; Data.Field rankField = dbProvider.GetField("Rank"); int tab = rankField.TabIndex; Stopwatch sw = new Stopwatch(); //int docid = rand.Next(lastDocId); sw.Start(); for (int j = 0; j < count / payloads.Length; j++) { dbProvider.FillPayloadRank(tab, count, payloads); } sw.Stop(); OutputValue("Times", count); OutputValue("Elapse(ms)", sw.ElapsedMilliseconds); OutputValue("ElapseOneTime(ms)", (double)sw.ElapsedMilliseconds / count); }
void ShowValue(string tableName) { Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist.", tableName)); } AddColumn("TableName"); AddColumn("QueryCacheEnabled"); AddColumn("QueryCacheTimeout"); OutputValue("TableName", tableName); OutputValue("QueryCacheEnabled", dbProvider.QueryCacheEnabled.ToString()); OutputValue("QueryCacheTimeout", dbProvider.QueryCacheTimeout.ToString()); }
public void Run() { Global.UserRightProvider.CanDo(Right.RightItem.Select); if (Parameters.Count < 2) { throw new ArgumentException("Parameter 1 is select statement before where. Parameter 2 - n is docids."); } string sql = Parameters[0]; SFQL.Parse.SFQLParse sfqlParse = new Hubble.Core.SFQL.Parse.SFQLParse(); sfqlParse.SyntaxAnalyse(sql); SFQL.SyntaxAnalysis.Select.Select select = sfqlParse.SFQLSentenceList[0].SyntaxEntity as SFQL.SyntaxAnalysis.Select.Select; string tableName = select.SelectFroms[0].Name; Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new Data.DataException(string.Format("Invalid table:{0}", tableName)); } List <Data.Field> fields; int allFieldsCount; sfqlParse.GetSelectFields(select, dbProvider, out fields, out allFieldsCount); Query.DocumentResultForSort[] docs = new Hubble.Core.Query.DocumentResultForSort[Parameters.Count - 1]; for (int i = 1; i < Parameters.Count; i++) { docs[i - 1] = new Hubble.Core.Query.DocumentResultForSort(int.Parse(Parameters[i])); } List <Data.Document> docResult = dbProvider.Query(fields, docs); Hubble.Framework.Data.DataSet ds = Data.Document.ToDataSet(fields, docResult); ds.Tables[0].TableName = "Select_" + select.SelectFroms[0].Alias; _QueryResult.DataSet = ds; }
public void Run() { Global.UserRightProvider.CanDo(Right.RightItem.ManageDB); if (Parameters.Count != 1) { throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name"); } Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0])); } dbProvider.StopSynchronize(); OutputMessage(string.Format("Table: {0}'s synchronization is stopping now!", Parameters[0])); }
public void Run() { if (Parameters.Count != 1) { throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name."); } Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0], false); if (dbProvider == null) { throw new Data.DataException(string.Format("Table name {0} does not exist!", Parameters[0])); //OutputMessage(string.Format("Table name {0} does not exist!", Parameters[0])); } else { AddColumn("FieldName"); AddColumn("DataType"); AddColumn("DataLength"); AddColumn("IndexType"); AddColumn("Analyzer"); AddColumn("IsNull"); AddColumn("IsPrimaryKey"); AddColumn("Default"); foreach (Data.Field field in dbProvider.GetAllFields()) { NewRow(); OutputValue("FieldName", field.Name); OutputValue("DataType", field.DataType.ToString()); OutputValue("DataLength", field.DataLength); OutputValue("IndexType", field.IndexType.ToString()); OutputValue("Analyzer", field.AnalyzerName); OutputValue("IsNull", field.CanNull); OutputValue("IsPrimaryKey", field.PrimaryKey); OutputValue("Default", field.DefaultValue); } } }
public void Run() { Global.UserRightProvider.CanDo(Right.RightItem.DropTable); if (Parameters.Count != 1) { throw new ArgumentException("the number of parameters must be 1. Parameter 1 is table name."); } Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0], false); if (dbProvider == null) { OutputMessage(string.Format("Table name {0} does not exist!", Parameters[0])); } else { Data.DBProvider.Drop(Parameters[0]); } OutputMessage(string.Format("Drop table {0} successul.", Parameters[0])); }
public void Run() { Global.UserRightProvider.CanDo(Right.RightItem.ManageDB); if (Parameters.Count != 1) { throw new ArgumentException("the number of parameters must be 1. Parameter 1 is tableName"); } string tableName = Parameters[0]; Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0])); } dbProvider.Detach(); OutputMessage(string.Format("Detach table {0} successul.", tableName)); }
public void Run() { if (Parameters.Count != 1) { throw new StoredProcException("First parameter is table name."); } Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0])); } AddColumn("Progress"); AddColumn("InsertRows"); NewRow(); OutputValue("Progress", dbProvider.TableSynchronizeProgress); OutputValue("InsertRows", dbProvider.TableSynchronizeInsertRows); }
void SetValue(string tableName, string value) { Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist.", tableName)); } bool indexonly; if (bool.TryParse(value, out indexonly)) { dbProvider.SetIndexOnly(indexonly); dbProvider.SaveTable(); OutputMessage(string.Format("Set table {0} index only to {1} sucessful!", tableName, dbProvider.IndexOnly)); } else { throw new StoredProcException("Parameter 2 must be 'True' or 'False'"); } }
public static DocId2LongComparer Generate(Data.DBProvider dbProvider, OrderBy[] orderBys, out Data.Field[] orderByFields) { bool[] ascs = new bool[orderBys.Length]; orderByFields = new Hubble.Core.Data.Field[orderBys.Length]; int scoreFieldIndex = -1; for (int i = 0; i < ascs.Length; i++) { if (orderBys[i].Order == null) { ascs[i] = true; } else { ascs[i] = !orderBys[i].Order.Equals("desc", StringComparison.CurrentCultureIgnoreCase); } if (orderBys[i].Name.Equals("docid", StringComparison.CurrentCultureIgnoreCase)) { orderByFields[i] = new Hubble.Core.Data.Field("docid", Hubble.Core.Data.DataType.Int); } else if (orderBys[i].Name.Equals("score", StringComparison.CurrentCultureIgnoreCase)) { scoreFieldIndex = i; orderByFields[i] = new Hubble.Core.Data.Field("score", Hubble.Core.Data.DataType.BigInt); } else { orderByFields[i] = dbProvider.GetField(orderBys[i].Name); } } return(new DocId2LongComparer(ascs, scoreFieldIndex)); }
public WordIndexReader(string word, WordStepDocIndex wordStepDocIndex, int totalDocs, Data.DBProvider dbProvider, IndexFileProxy indexProxy, int maxReturnCount) { _Word = word; _IndexReader = new IndexReader(wordStepDocIndex, indexProxy); _DBProvider = dbProvider; _DelProvider = _DBProvider.DelProvider; //_TabIndex = tabIndex; _TotalDocs = totalDocs; _WordCountSum = wordStepDocIndex.WordCountSum; _RelDocCount = wordStepDocIndex.RelDocCount; if (maxReturnCount < 0) { _Count = _RelDocCount; } else { _Count = Math.Min(maxReturnCount, _RelDocCount); } //UpdateDocScoreList(); }
public void Run() { if (Parameters.Count != 1) { throw new StoredProcException("First parameter is table name."); } Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0])); } AddColumn("Attribute"); AddColumn("Value"); NewRow(); OutputValue("Attribute", "Directory"); OutputValue("Value", dbProvider.Directory); NewRow(); OutputValue("Attribute", "IndexOnly"); OutputValue("Value", dbProvider.IndexOnly.ToString()); NewRow(); OutputValue("Attribute", "DocId"); OutputValue("Value", dbProvider.DocIdReplaceField); NewRow(); OutputValue("Attribute", "DBTableName"); OutputValue("Value", dbProvider.Table.DBTableName); NewRow(); OutputValue("Attribute", "DBAdapter"); OutputValue("Value", ((Data.INamedExternalReference)dbProvider.DBAdapter).Name); NewRow(); OutputValue("Attribute", "LastDocId"); OutputValue("Value", dbProvider.LastDocIdForIndexOnly.ToString()); NewRow(); OutputValue("Attribute", "MaxReturnCount"); OutputValue("Value", dbProvider.MaxReturnCount.ToString()); NewRow(); OutputValue("Attribute", "GroupByLimit"); OutputValue("Value", dbProvider.Table.GroupByLimit.ToString()); NewRow(); OutputValue("Attribute", "InitImmediatelyAfterStartup"); OutputValue("Value", dbProvider.Table.InitImmediatelyAfterStartup.ToString()); NewRow(); OutputValue("Attribute", "RamIndexType"); OutputValue("Value", dbProvider.Table.RamIndexType.ToString()); NewRow(); OutputValue("Attribute", "RamIndexMinLoadSize"); OutputValue("Value", dbProvider.Table.RamIndexMinLoadSize.ToString()); NewRow(); OutputValue("Attribute", "QueryCacheEnabled"); OutputValue("Value", dbProvider.QueryCacheEnabled.ToString()); NewRow(); OutputValue("Attribute", "QueryCacheTimeout"); OutputValue("Value", dbProvider.QueryCacheTimeout.ToString()); NewRow(); OutputValue("Attribute", "StoreQueryCacheInFile"); OutputValue("Value", dbProvider.Table.StoreQueryCacheInFile.ToString()); NewRow(); OutputValue("Attribute", "CleanupQueryCacheFileInDays"); OutputValue("Value", dbProvider.Table.CleanupQueryCacheFileInDays.ToString()); NewRow(); OutputValue("Attribute", "IndexThread"); OutputValue("Value", dbProvider.Table.IndexThread.ToString()); NewRow(); OutputValue("Attribute", "Debug"); OutputValue("Value", dbProvider.Table.Debug.ToString()); NewRow(); OutputValue("Attribute", "TableSynchronization"); OutputValue("Value", dbProvider.Table.TableSynchronization.ToString()); NewRow(); OutputValue("Attribute", "TriggerTableName"); OutputValue("Value", dbProvider.Table.TriggerTableName); NewRow(); OutputValue("Attribute", "MirrorTableEnabled"); OutputValue("Value", dbProvider.Table.MirrorTableEnabled); NewRow(); OutputValue("Attribute", "UsingMirrorTableForNonFulltextQuery"); OutputValue("Value", dbProvider.Table.UsingMirrorTableForNonFulltextQuery); NewRow(); OutputValue("Attribute", "SelectTimeout"); OutputValue("Value", dbProvider.Table.SelectTimeout); }
public void Run() { Global.UserRightProvider.CanDo(Right.RightItem.ManageDB); if (Parameters.Count < 1) { throw new ArgumentException("the number of parameters must large than 0. Parameter 1 is table name, Parameter 2 is step, Parameter 3 is optimize mode"); } string tableName = Parameters[0]; Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0])); } if (dbProvider.TableSynchronizeProgress >= 0 && dbProvider.TableSynchronizeProgress < 100) { return; } int option = 0; int step = 5000; bool fastestMode = true; string strFlags = ""; Service.SyncFlags flags = Service.SyncFlags.Insert | Service.SyncFlags.Rebuild; if (Parameters.Count > 1) { step = int.Parse(Parameters[1]); } if (Parameters.Count > 2) { option = int.Parse(Parameters[2]); } if (Parameters.Count > 3) { strFlags = Parameters[3]; flags = (Service.SyncFlags)Hubble.Framework.Reflection.Emun.FromString(typeof(Service.SyncFlags), strFlags); flags &= ~Hubble.Core.Service.SyncFlags.Delete; flags &= ~Hubble.Core.Service.SyncFlags.Update; flags |= Hubble.Core.Service.SyncFlags.Rebuild; flags |= Hubble.Core.Service.SyncFlags.Insert; } Hubble.Core.Data.OptimizationOption optimizeOption; switch (option) { case 0: optimizeOption = Hubble.Core.Data.OptimizationOption.Idle; break; case 1: optimizeOption = Hubble.Core.Data.OptimizationOption.Minimum; break; case 2: optimizeOption = Hubble.Core.Data.OptimizationOption.Middle; break; case 3: optimizeOption = Hubble.Core.Data.OptimizationOption.Speedy; break; default: optimizeOption = Hubble.Core.Data.OptimizationOption.Minimum; break; } bool notIndexOnly = false; try { if (!dbProvider.IndexOnly) { notIndexOnly = true; dbProvider.Table.IndexOnly = true; } Data.DBProvider.Truncate(tableName); if (notIndexOnly) { dbProvider = Data.DBProvider.GetDBProvider(tableName); dbProvider.Table.IndexOnly = false; } } catch { if (notIndexOnly) { dbProvider = Data.DBProvider.GetDBProvider(tableName); dbProvider.Table.IndexOnly = false; } } dbProvider = Data.DBProvider.GetDBProvider(tableName); dbProvider.SynchronizeWithDatabase(step, optimizeOption, fastestMode, flags); if ((flags & Hubble.Core.Service.SyncFlags.WaitForExit) != 0) { //Wait for exit while (true) { if (dbProvider.TableSynchronizeProgress >= 100 || dbProvider.TableSynchronizeProgress < 0) { OutputMessage(string.Format("Table: {0} finished synchronizing!", Parameters[0])); return; } System.Threading.Thread.Sleep(1000); } } OutputMessage(string.Format("Table: {0} is synchronizing now!", Parameters[0])); }
unsafe private string GetInSql(Data.DBProvider dbProvider, Query.DocumentResultForSort[] result, int begin, int count) { if (begin + count > result.Length) { return(null); } StringBuilder sql = new StringBuilder(); if (dbProvider.DocIdReplaceField == null) { sql.Append("docId in ("); } else { sql.AppendFormat("{0} in (", dbProvider.DocIdReplaceField); } Dictionary <long, int> replaceFieldValueToDocId = null; if (dbProvider.DocIdReplaceField != null) { replaceFieldValueToDocId = new Dictionary <long, int>(); } int i = 0; for (int j = begin; j < begin + count; j++) { Query.DocumentResultForSort docResult = result[j]; int docId = docResult.DocId; if (dbProvider.DocIdReplaceField == null) { if (i++ == 0) { sql.AppendFormat("{0}", docId); } else { sql.AppendFormat(",{0}", docId); } } else { long replaceFieldValue = dbProvider.GetDocIdReplaceFieldValue(docId); if (i++ == 0) { sql.AppendFormat("{0}", replaceFieldValue); } else { sql.AppendFormat(",{0}", replaceFieldValue); } } } sql.Append(")"); return(sql.ToString()); }
public void Run() { if (Parameters.Count != 3) { throw new StoredProcException("First parameter is table name, second parameter is field name, third parameter is words. SP_GetIDF 'tablename', 'fieldname', 'abc news'"); } string tableName = Parameters[0]; string fieldName = Parameters[1]; Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(tableName); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", tableName)); } Hubble.Core.Index.InvertedIndex invertedIndex = dbProvider.GetInvertedIndex(fieldName); if (invertedIndex == null) { throw new StoredProcException(string.Format("Field name {0} does not exist or is not the tokenized index field!", fieldName)); } string queryStr = Parameters[2]; List <WordInfo> wordInfos = ParseWhere.GetWordInfoList(queryStr); Dictionary <string, WordIndexForQuery> wordIndexDict = new Dictionary <string, WordIndexForQuery>(); foreach (Hubble.Core.Entity.WordInfo wordInfo in wordInfos) { WordIndexForQuery wifq; if (!wordIndexDict.TryGetValue(wordInfo.Word, out wifq)) { //Hubble.Core.Index.WordIndexReader wordIndex = InvertedIndex.GetWordIndex(wordInfo.Word, CanLoadPartOfDocs); //Get whole index Hubble.Core.Index.WordIndexReader wordIndex = invertedIndex.GetWordIndex(wordInfo.Word, false, true); //Only get step doc index if (wordIndex == null) { wordIndexDict.Add(wordInfo.Word, null); continue; } wifq = new WordIndexForQuery(wordIndex, invertedIndex.DocumentCount, wordInfo.Rank, 1); wifq.QueryCount = 1; wifq.FirstPosition = wordInfo.Position; wordIndexDict.Add(wordInfo.Word, wifq); } else { wifq.WordRank += wordInfo.Rank; wifq.QueryCount++; } //wordIndexList[wordIndexList.Count - 1].Rank += wordInfo.Rank; } AddColumn("Word"); AddColumn("TF"); AddColumn("IDF"); AddColumn("T_D"); AddColumn("TotalDoucments"); AddColumn("TF_IDF"); int totalDocuments = invertedIndex.DocumentCount; foreach (string word in wordIndexDict.Keys) { NewRow(); WordIndexForQuery wifq = wordIndexDict[word]; OutputValue("Word", word); if (wifq == null) { OutputValue("TF", 0); OutputValue("T_D", 0); OutputValue("TotalDoucments", 0); OutputValue("IDF", 0); OutputValue("TF_IDF", 0); } else { double idf = Math.Log((double)totalDocuments / (double)wifq.RelTotalCount); OutputValue("TF", wifq.QueryCount); OutputValue("T_D", wifq.RelTotalCount); OutputValue("TotalDoucments", totalDocuments); OutputValue("IDF", idf); OutputValue("TF_IDF", wifq.QueryCount * idf); } } }
public void Run() { Global.UserRightProvider.CanDo(Right.RightItem.ManageDB); if (Parameters.Count < 1) { throw new ArgumentException("the number of parameters must large than 0. Parameter 1 is table name, Parameter 2 is step, Parameter 3 is optimize mode"); } Data.DBProvider dbProvider = Data.DBProvider.GetDBProvider(Parameters[0]); if (dbProvider == null) { throw new StoredProcException(string.Format("Table name {0} does not exist!", Parameters[0])); } int option = 0; int step = 5000; bool fastestMode = false; string strFlags = ""; Service.SyncFlags flags = Service.SyncFlags.Insert | Service.SyncFlags.Delete | Service.SyncFlags.Update; if (Parameters.Count > 1) { step = int.Parse(Parameters[1]); } if (Parameters.Count > 2) { option = int.Parse(Parameters[2]); } if (Parameters.Count > 3) { fastestMode = bool.Parse(Parameters[3]); } if (Parameters.Count > 4) { strFlags = Parameters[4]; flags = (Service.SyncFlags)Hubble.Framework.Reflection.Emun.FromString(typeof(Service.SyncFlags), strFlags); } Hubble.Core.Data.OptimizationOption optimizeOption; switch (option) { case 0: optimizeOption = Hubble.Core.Data.OptimizationOption.Idle; break; case 1: optimizeOption = Hubble.Core.Data.OptimizationOption.Minimum; break; case 2: optimizeOption = Hubble.Core.Data.OptimizationOption.Middle; break; case 3: optimizeOption = Hubble.Core.Data.OptimizationOption.Speedy; break; default: optimizeOption = Hubble.Core.Data.OptimizationOption.Minimum; break; } dbProvider.SynchronizeWithDatabase(step, optimizeOption, fastestMode, flags); if ((flags & Hubble.Core.Service.SyncFlags.WaitForExit) != 0) { //Wait for exit int count = 0; while (true) { if (dbProvider.TableSynchronizeProgress >= 100 || dbProvider.TableSynchronizeProgress < 0) { OutputMessage(string.Format("Table: {0} finished synchronizing!", Parameters[0])); return; } if ((++count % 60) == 0) { Hubble.Core.Global.Report.WriteAppLog(string.Format("TableSynchronizeProgress = {0}% of table:{1}", dbProvider.TableSynchronizeProgress, dbProvider.TableName)); } System.Threading.Thread.Sleep(1000); } } OutputMessage(string.Format("Table: {0} is synchronizing now!", Parameters[0])); }
internal Hubble.Core.Index.WordIndexReader GetWordIndex(string word, WordFilePositionList filePositionList, int totalDocs, Data.DBProvider dbProvider, int maxReturnCount) { WordDocumentsList docList = new WordDocumentsList(); bool simple = _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple; if (maxReturnCount < 0) { foreach (FilePosition filePosition in filePositionList.FPList) { IDXFile idxFile = GetIDXFile(filePosition.Serial); if (idxFile == null) { continue; } WordDocumentsList wdl = idxFile.GetDocList(filePosition.Position, filePosition.Length, -1, simple); if (filePositionList.Count == 1) { docList = wdl; } else { docList.AddRange(wdl); docList.WordCountSum += wdl.WordCountSum; } } } else { int remain = maxReturnCount; foreach (FilePosition filePosition in filePositionList.FPList) { IDXFile idxFile = GetIDXFile(filePosition.Serial); if (idxFile == null) { continue; } WordDocumentsList wdl = idxFile.GetDocList(filePosition.Position, filePosition.Length, remain, simple); if (filePositionList.Count == 1) { docList = wdl; } else { docList.AddRange(wdl); docList.WordCountSum += wdl.WordCountSum; docList.RelDocCount += wdl.RelDocCount; } remain -= wdl.Count; } } return(new WordIndexReader(word, docList, totalDocs, dbProvider)); }
/// <summary> /// Only get the word step doc index. /// </summary> /// <param name="word"></param> /// <param name="filePositionList"></param> /// <param name="totalDocs"></param> /// <param name="dbProvider"></param> /// <param name="maxReturnCount"></param> /// <returns></returns> internal Hubble.Core.Index.WordIndexReader GetWordIndexWithWordStepDocIndex(string word, WordFilePositionList filePositionList, int totalDocs, Data.DBProvider dbProvider, int maxReturnCount) { List <WordStepDocIndex.IndexFileInfo> indexFileInfoList = new List <WordStepDocIndex.IndexFileInfo>(); //StepDocIndex of all index files bool simple = _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple; long wordCountSum = 0; foreach (FilePosition filePosition in filePositionList.FPList) { IDXFile idxFile = GetIDXFile(filePosition.Serial); if (idxFile == null) { continue; } int count; long indexPosition; List <Hubble.Core.Entity.DocumentPosition> stepDocIndex = idxFile.GetStepDocIndex(filePosition.Position, filePosition.Length, out count, out indexPosition); indexFileInfoList.Add(new WordStepDocIndex.IndexFileInfo(stepDocIndex, new FilePosition(filePosition.Serial, indexPosition, (int)(filePosition.Length - (indexPosition - filePosition.Position) - sizeof(int))), //Subtract last 4 bytes for last doc id and Subtract stepdocindex length count)); wordCountSum += count; } int relDocCount = (int)wordCountSum; //if (maxReturnCount >= 0) //{ // if (wordCountSum > maxReturnCount) // { // relDocCount = maxReturnCount; // } //} WordStepDocIndex wordStepDocIndex = new WordStepDocIndex(indexFileInfoList, wordCountSum, relDocCount, simple); return(new WordIndexReader(word, wordStepDocIndex, totalDocs, dbProvider, _IndexFileProxy, maxReturnCount)); }
/// <summary> /// Get IdFields List from trigger table /// </summary> /// <param name="dbProvider">dbProvider that used to get field info</param> /// <param name="dbAdapterName">DBAdapter name</param> /// <param name="table">table that read from trigger table</param> /// <param name="lastSerial">the last serial number of trigger table from which read</param> /// <returns></returns> internal static List <IdFields> GetIdFieldsList(Data.DBProvider dbProvider, string dbAdapterName, System.Data.DataTable table, out long lastSerial) { lastSerial = -1; HashSet <string> fieldsSetWithTokenizedFields = new HashSet <string>(); HashSet <string> fieldsSetWithoutTokenizedFields = new HashSet <string>(); List <string> tempFields = new List <string>(128); List <IdFields> result = new List <IdFields>(table.Rows.Count); foreach (System.Data.DataRow row in table.Rows) { long id = long.Parse(row["id"].ToString()); lastSerial = long.Parse(row["Serial"].ToString()); string fields = row["Fields"].ToString(); tempFields.Clear(); bool hasTokenized = false; //check fields foreach (string field in fields.Split(new char[] { ',' })) { string f = field.Trim().ToLower(); if (f == "") { continue; } Data.Field dbField = dbProvider.GetField(f); if (dbField == null) { continue; } if (dbField.IndexType == Hubble.Core.Data.Field.Index.Tokenized) { hasTokenized = true; } tempFields.Add(f); } //Fill hash set if (hasTokenized) { foreach (string field in tempFields) { if (!fieldsSetWithTokenizedFields.Contains(field)) { fieldsSetWithTokenizedFields.Add(field); } } } else { foreach (string field in tempFields) { if (!fieldsSetWithoutTokenizedFields.Contains(field)) { fieldsSetWithoutTokenizedFields.Add(field); } } } result.Add(new IdFields(id, hasTokenized)); } //Get new fields string string fieldsWithTokenized = GetFieldsStringFromHashSet(fieldsSetWithTokenizedFields, dbAdapterName); string fieldsWithoutTokenized = GetFieldsStringFromHashSet(fieldsSetWithoutTokenizedFields, dbAdapterName); foreach (IdFields idFields in result) { if (idFields.HasTokenizedFields) { idFields.Fields = fieldsWithTokenized; } else { idFields.Fields = fieldsWithoutTokenized; } } //Merge same Id result.Sort(); if (result.Count > 0) { IdFields last = result[0]; for (int i = 1; i < result.Count; i++) { if (result[i].Equals(last)) { result[i] = null; continue; } else { last = result[i]; } } } return(result); }