public Core.SFQL.Parse.DocumentResultWhereDictionary GetDocumentResults(int end, string where, string orderby) { string sql; if (end >= 0) { sql = string.Format("select top {0} ", end + 1); } else { sql = "select "; } if (string.IsNullOrEmpty(where)) { if (DocIdReplaceField == null) { sql += string.Format(" docid from [{0}] ", Table.DBTableName); } else { sql += string.Format(" [{0}] from [{1}] ", DocIdReplaceField, Table.DBTableName); } } else { if (DocIdReplaceField == null) { sql += string.Format(" docid from [{0}] where {1}", Table.DBTableName, where); } else { sql += string.Format(" [{0}] from [{1}] where {2}", DocIdReplaceField, Table.DBTableName, where); } } if (!string.IsNullOrEmpty(orderby)) { sql += " order by " + orderby; } Core.SFQL.Parse.DocumentResultWhereDictionary result = new Core.SFQL.Parse.DocumentResultWhereDictionary(); using (SQLDataProvider sqlData = new SQLDataProvider()) { sqlData.Connect(Table.ConnectionString); foreach (System.Data.DataRow row in sqlData.QuerySql(sql).Tables[0].Rows) { int docId; if (DocIdReplaceField == null) { docId = int.Parse(row[0].ToString()); } else { docId = DBProvider.GetDocIdFromDocIdReplaceFieldValue(long.Parse(row[DocIdReplaceField].ToString())); if (docId < 0) { continue; } } result.Add(docId, new Hubble.Core.Query.DocumentResult(docId)); } System.Data.DataSet ds; if (string.IsNullOrEmpty(where)) { ds = sqlData.QuerySql(string.Format("select count(*) cnt from {0}", Table.DBTableName)); } else { ds = sqlData.QuerySql(string.Format("select count(*) cnt from {0} where {1}", Table.DBTableName, where)); } result.RelTotalCount = int.Parse(ds.Tables[0].Rows[0][0].ToString()); } return(result); }
/// <summary> /// Order by score desc /// and only one expression in the banch of expression tree. /// and more than two words /// </summary> /// <param name="upDict"></param> /// <param name="docIdRank"></param> /// <param name="wordIndexes"></param> unsafe private void CalculateWithPositionOrderByScoreDesc11(Core.SFQL.Parse.DocumentResultWhereDictionary upDict, ref Core.SFQL.Parse.DocumentResultWhereDictionary docIdRank, WordIndexForQuery[] wordIndexes) { if (upDict != null) { throw new ParseException("UpDict is not null!"); } Array.Sort(wordIndexes); //Calculate top int top; if (this._QueryParameter.End >= 0) { top = (1 + this._QueryParameter.End / 100) * 100; if (top <= 0) { top = 100; } //if (this._QueryParameter.End * 2 > top) //{ // top *= 2; //} } else { top = int.MaxValue; } double ratio = 1; if (wordIndexes.Length > 1) { ratio = (double)2 / (double)(wordIndexes.Length - 1); } //Get max word doc list count int maxWordDocListCount = 0; int documentSum = 0; foreach (WordIndexForQuery wifq in wordIndexes) { maxWordDocListCount += wifq.WordIndex.Count; } maxWordDocListCount += maxWordDocListCount / 2; if (maxWordDocListCount > 1024 * 1024) { maxWordDocListCount = 1024 * 1024; } Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Calculate"); bool groupbyScanAll = false; //Match for group by if (this._QueryParameter.NeedGroupBy) { groupbyScanAll = true; int groupbyContainsCount = 0; int groupbyLimit = _DBProvider.Table.GroupByLimit; BitSet bitSet = new BitSet(); for (int i = 0; i < wordIndexes.Length; i++) { WordIndexForQuery wifq = wordIndexes[i]; Entity.DocumentPositionList docList = wifq.WordIndex.GetNext(); while (docList.DocumentId >= 0) { if (bitSet.ForceAdd(docList.DocumentId)) { groupbyContainsCount++; } if (groupbyContainsCount >= groupbyLimit) { groupbyScanAll = false; break; } docList = wifq.WordIndex.GetNext(); } wifq.WordIndex.Reset(); if (!groupbyScanAll) { break; } } AscIntList groupByCollect = new AscIntList(); groupByCollect.AddRange(bitSet); } //Merge int indexInTop = 0; for (int i = 0; i < wordIndexes.Length; i++) { if (docIdRank.Count >= top) { break; } indexInTop = i; WordIndexForQuery wifq = wordIndexes[i]; //Entity.DocumentPositionList[] wifqDocBuf = wifq.WordIndex.DocPositionBuf; Entity.DocumentPositionList docList = wifq.WordIndex.GetNext(); int j = 0; while (docList.DocumentId >= 0) { Core.SFQL.Parse.DocumentResultPoint drp; drp.pDocumentResult = null; long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)docList.Count * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } else { switch (i) { case 0: score *= 20; break; case 1: score *= 4; break; case 2: score *= 1; break; case 3: score /= 2; break; default: score /= i; break; } } if (score < 0) { //Overflow score = long.MaxValue - 4000000; } bool exits = drp.pDocumentResult != null; if (!exits && i > 0) { exits = docIdRank.TryGetValue(docList.DocumentId, out drp); } if (exits) { drp.pDocumentResult->Score += score; drp.pDocumentResult->HitCount++; double queryPositionDelta = wifq.FirstPosition - drp.pDocumentResult->LastWordIndexFirstPosition; double positionDelta = docList.FirstPosition - drp.pDocumentResult->LastPosition; double delta = Math.Abs(queryPositionDelta - positionDelta); if (delta < 0.031) { delta = 0.031; } else if (delta <= 1.1) { delta = 0.5; } else if (delta <= 2.1) { delta = 1; } delta = Math.Pow((1 / delta), ratio) * docList.Count * drp.pDocumentResult->LastCount / (double)(wifq.QueryCount * drp.pDocumentResult->LastWordIndexQueryCount); drp.pDocumentResult->Score = (long)(drp.pDocumentResult->Score * delta); //Overflow, if match too much, sometime score would less than zero. if (drp.pDocumentResult->Score < 0) { drp.pDocumentResult->Score = long.MaxValue - 4000000; } drp.pDocumentResult->LastIndex = (UInt16)i; drp.pDocumentResult->LastPosition = docList.FirstPosition; drp.pDocumentResult->LastCount = (UInt16)docList.Count; drp.pDocumentResult->LastWordIndexFirstPosition = (UInt16)wifq.FirstPosition; } else { bool notInDict = false; if (_NotInDict != null) { if (_NotInDict.ContainsKey(docList.DocumentId)) { notInDict = true; } } if (!notInDict) { //upDict is null in this function DocumentResult docResult = new DocumentResult(docList.DocumentId, score, wifq.FirstPosition, wifq.QueryCount, docList.FirstPosition, docList.Count, i); docIdRank.Add(docList.DocumentId, docResult); } } docList = wifq.WordIndex.GetNext(); j++; if (j > wifq.WordIndex.Count) { break; } } } long maxScoreValue = 0; //Max score value of the docid that hit count less than wordIndexes.Length int wordIndexesLen = wordIndexes.Length; //Get the max score value of the docs that hit count less than wordIndexes.Length foreach (DocumentResultPoint docResult in docIdRank.Values) { if (docResult.pDocumentResult->HitCount < wordIndexesLen) { if (docResult.pDocumentResult->Score > maxScoreValue) { maxScoreValue = docResult.pDocumentResult->Score; } } } double hitRate = 0; if (indexInTop < wordIndexes.Length - 1) { int[] docidlist = new int[docIdRank.Count]; int i = 0; foreach (int docid in docIdRank.Keys) { docidlist[i] = docid; i++; } Array.Sort(docidlist); int lastWordHitCount = 0; foreach (int firstDocId in docidlist) { int curWord = indexInTop + 1; Core.SFQL.Parse.DocumentResultPoint drp; if (docIdRank.TryGetValue(firstDocId, out drp)) { while (curWord < wordIndexesLen) { Entity.DocumentPositionList docList = wordIndexes[curWord].WordIndex.Get(firstDocId); int curDocId = docList.DocumentId; if (curDocId >= 0) { drp.pDocumentResult->HitCount++; if (curWord == wordIndexesLen - 1) { lastWordHitCount++; } WordIndexForQuery wifq = wordIndexes[curWord]; long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)docList.Count * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } else { switch (curWord) { case 0: score *= 20; break; case 1: score *= 4; break; case 2: score *= 1; break; case 3: score /= 2; break; default: score /= curWord; break; } } if (score < 0) { //Overflow score = long.MaxValue - 4000000; } drp.pDocumentResult->Score += score; double queryPositionDelta = wifq.FirstPosition - drp.pDocumentResult->LastWordIndexFirstPosition; double positionDelta = docList.FirstPosition - drp.pDocumentResult->LastPosition; double delta = Math.Abs(queryPositionDelta - positionDelta); if (delta < 0.031) { delta = 0.031; } else if (delta <= 1.1) { delta = 0.5; } else if (delta <= 2.1) { delta = 1; } delta = Math.Pow((1 / delta), ratio) * docList.Count * drp.pDocumentResult->LastCount / (double)(wifq.QueryCount * drp.pDocumentResult->LastWordIndexQueryCount); drp.pDocumentResult->Score = (long)(drp.pDocumentResult->Score * delta); //Overflow, if match too much, sometime score would less than zero. if (drp.pDocumentResult->Score < 0) { drp.pDocumentResult->Score = long.MaxValue - 4000000; } drp.pDocumentResult->LastIndex = (UInt16)curWord; drp.pDocumentResult->LastPosition = docList.FirstPosition; drp.pDocumentResult->LastCount = (UInt16)docList.Count; drp.pDocumentResult->LastWordIndexFirstPosition = (UInt16)wifq.FirstPosition; } curWord++; } //While if (drp.pDocumentResult->HitCount < wordIndexesLen) { if (drp.pDocumentResult->Score > maxScoreValue) { maxScoreValue = drp.pDocumentResult->Score; } } } } if (docidlist.Length > 0) { hitRate = (double)lastWordHitCount / (double)docidlist.Length; } } //Adjust score of the docs that hit count equal wordIndexes.Length foreach (DocumentResultPoint docResult in docIdRank.Values) { if (docResult.pDocumentResult->HitCount == wordIndexesLen) { docResult.pDocumentResult->Score += maxScoreValue; if (docResult.pDocumentResult->Score < 0) { docResult.pDocumentResult->Score = long.MaxValue; } } } performanceReport.Stop(); documentSum += docIdRank.Count; if (indexInTop < wordIndexes.Length - 1) { documentSum += wordIndexes[wordIndexes.Length - 1].RelTotalCount; if (hitRate > 0) { int predictCount = 0; for (int i = indexInTop + 1; i < wordIndexes.Length - 1; i++) { predictCount += (int)(wordIndexes[i].RelTotalCount * (1 - hitRate)); } documentSum += predictCount; } } if (documentSum > _TotalDocuments) { documentSum = _TotalDocuments; } docIdRank.RelTotalCount = documentSum; DeleteProvider delProvider = _DBProvider.DelProvider; int deleteCount = delProvider.Filter(docIdRank); docIdRank.RelTotalCount -= deleteCount; if (groupbyScanAll) { docIdRank.RelTotalCount = docIdRank.GroupByCollection.Count; } else if (docIdRank.GroupByCollection.Count > docIdRank.RelTotalCount) { docIdRank.RelTotalCount = docIdRank.GroupByCollection.Count; } }
unsafe private void CalculateWithPositionOrderByScoreDesc(Core.SFQL.Parse.DocumentResultWhereDictionary upDict, ref Core.SFQL.Parse.DocumentResultWhereDictionary docIdRank, WordIndexForQuery[] wordIndexes) { DBProvider dbProvider = Argument.DBProvider; bool needFilterUntokenizedConditions = this.Argument.NeedFilterUntokenizedConditions; ExpressionTree untokenizedTree = this.Argument.UntokenizedTreeOnRoot; if (upDict != null) { throw new ParseException("UpDict is not null!"); } //Calculate top int top; if (Argument.End >= 0) { top = (1 + Argument.End / 100) * 100; if (top <= 0) { top = 100; } } else { top = int.MaxValue; } PriorQueue <Docid2Long> priorQueue = null; List <Docid2Long> docid2longList = null; if (top == int.MaxValue) { docid2longList = new List <Docid2Long>(); } else { priorQueue = new PriorQueue <Docid2Long>(top, new DocIdLongComparer(false)); } long lastMinScore = 0; int rows = 0; Core.SFQL.Parse.DocumentResultWhereDictionary groupByDict = Argument.NeedGroupBy ? docIdRank : null; MultiWordsDocIdEnumerator mwde = new MultiWordsDocIdEnumerator(wordIndexes, dbProvider, groupByDict, -1, needFilterUntokenizedConditions); //Changed at 2012-3-18, top optimize will effect search result, disable it. //MultiWordsDocIdEnumerator mwde = new MultiWordsDocIdEnumerator(wordIndexes, dbProvider, groupByDict, top, // needFilterUntokenizedConditions); Entity.OriginalDocumentPositionList odpl = new Hubble.Core.Entity.OriginalDocumentPositionList(); mwde.GetNextOriginal(ref odpl); Entity.DocumentPositionList lastDocList = new Hubble.Core.Entity.DocumentPositionList(); double ratio = 1; if (wordIndexes.Length > 1) { ratio = (double)2 / (double)(wordIndexes.Length - 1); } Query.DocumentResult documentResult; Query.DocumentResult *drp = &documentResult; int skipCount = 0; //skip by filter untokenized conditions while (odpl.DocumentId >= 0) { //Process untokenized conditions. //If is not matched, get the next one. if (needFilterUntokenizedConditions) { int docId = odpl.DocumentId; drp->DocId = docId; drp->PayloadData = dbProvider.GetPayloadDataWithShareLock(docId); if (!ParseWhere.GetComparisionExpressionValue(dbProvider, drp, untokenizedTree)) { mwde.GetNextOriginal(ref odpl); skipCount++; continue; } } //Matched //Caculate score #region Caclate score long totalScore = 0; lastDocList.Count = 0; lastDocList.FirstPosition = 0; int lastWifqIndex = 0; for (int i = 0; i < mwde.SelectedCount; i++) { int index = mwde.SelectedIndexes[i]; WordIndexForQuery wifq = mwde.WordIndexes[index]; Int16 count = (Int16)mwde.SelectedDocLists[i].Count; int firstPosition = mwde.SelectedDocLists[i].FirstPosition; int totalWordsInThisDocument = mwde.SelectedDocLists[i].TotalWordsInThisDocument; long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)count * (long)1000000 / ((long)wifq.Sum_d_t * (long)totalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } double delta = 1; if (i > 0) { //Calculate with position double queryPositionDelta = wifq.FirstPosition - wordIndexes[lastWifqIndex].FirstPosition; double positionDelta = firstPosition - lastDocList.FirstPosition; delta = Math.Abs(queryPositionDelta - positionDelta); if (delta < 0.031) { delta = 0.031; } else if (delta <= 1.1) { delta = 0.5; } else if (delta <= 2.1) { delta = 1; } delta = Math.Pow((1 / delta), ratio) * count * lastDocList.Count / (double)(wifq.QueryCount * wordIndexes[lastWifqIndex].QueryCount); } lastDocList.Count = count; lastDocList.FirstPosition = firstPosition; lastWifqIndex = index; totalScore += (long)(score * delta); } //End of score calculation if (_HasRankField) { int rank = dbProvider.SharedPayloadProvider.GetPayloadRank(odpl.DocumentId); totalScore *= rank; if (totalScore < 0) { totalScore = long.MaxValue - 4000000; } } //all of the words matched //10 times if (mwde.SelectedCount == wordIndexes.Length) { totalScore *= 10; if (totalScore < 0) { totalScore = long.MaxValue - 4000000; } } #endregion //Insert to prior queue if (rows >= top) { if (lastMinScore < totalScore) { priorQueue.Add(new Docid2Long(odpl.DocumentId, totalScore)); lastMinScore = priorQueue.Last.Value1; } } else { if (top == int.MaxValue) { docid2longList.Add(new Docid2Long(odpl.DocumentId, totalScore)); } else { priorQueue.Add(new Docid2Long(odpl.DocumentId, totalScore)); rows++; if (rows == top) { lastMinScore = priorQueue.Last.Value1; } } } mwde.GetNextOriginal(ref odpl); } docIdRank.RelTotalCount = mwde.TotalDocIdCount - skipCount; Docid2Long[] docid2longArr; if (top == int.MaxValue) { docid2longList.Sort(new DocIdLongComparer(false)); docid2longArr = docid2longList.ToArray(); } else { docid2longArr = priorQueue.ToArray(); } foreach (Docid2Long docid2Long in docid2longArr) { long score = docid2Long.Value1; if (score < 0) { //Overflow score = long.MaxValue - 4000000; } docIdRank.Add(docid2Long.DocId, new DocumentResult(docid2Long.DocId, score)); } docIdRank.Sorted = true; }
/// <summary> /// order by except only order by score desc. /// </summary> /// <param name="upDict"></param> /// <param name="docIdRank"></param> unsafe public void CalculateOptimizeNormalOrderBy(Core.SFQL.Parse.DocumentResultWhereDictionary upDict, ref Core.SFQL.Parse.DocumentResultWhereDictionary docIdRank) { DBProvider dBProvider = Argument.DBProvider; Argument.DBProvider.SharedPayloadProvider.EnterPayloladShareLock(); bool needFilterUntokenizedConditions = this.Argument.NeedFilterUntokenizedConditions; ExpressionTree untokenizedTree = this.Argument.UntokenizedTreeOnRoot; Query.DocumentResult documentResult; Query.DocumentResult *drp = &documentResult; bool orderByIncludingScore = Argument.OrderByIncludingScore(); try { Field[] orderByFields; DocId2LongComparer comparer = DocId2LongComparer.Generate( dBProvider, Argument.OrderBys, out orderByFields); bool needGroupBy = Argument.NeedGroupBy; WordIndexForQuery wifq = WordIndexes[0]; _IndexReader = wifq.WordIndex.IndexReader; Data.Field rankField = Argument.DBProvider.GetField("Rank"); if (rankField != null) { if (rankField.DataType == Hubble.Core.Data.DataType.Int && rankField.IndexType == Hubble.Core.Data.Field.Index.Untokenized) { _HasRandField = true; _RankTab = rankField.TabIndex; _DocidPayloads = new OriginalDocumentPositionList[2 * 1024]; _CurDocidPayloadIndex = _DocidPayloads.Length; } } if (_IndexReader != null) { int top; //vars for delete bool haveRecordsDeleted = dBProvider.DelProvider.Count > 0; int[] delDocs = null; int curDelIndex = 0; int curDelDocid = 0; int groupByCount = 0; int groupByLen = dBProvider.Table.GroupByLimit; int groupByStep = 1; int groupByIndex = 0; if (needGroupBy) { groupByStep = wifq.RelTotalCount / groupByLen; if (groupByStep <= 0) { groupByStep = 1; } } if (haveRecordsDeleted) { delDocs = dBProvider.DelProvider.DelDocs; curDelDocid = delDocs[curDelIndex]; } try { //calculate top //If less than 100, set to 100 if (this.Argument.End >= 0) { top = (1 + this.Argument.End / 100) * 100; if (top <= 0) { top = 100; } //if (this.Argument.End * 2 > top) //{ // top *= 2; //} } else { top = int.MaxValue; } PriorQueue <Docid2Long> priorQueue = new PriorQueue <Docid2Long>(top, comparer); int rows = 0; Entity.OriginalDocumentPositionList docList = new OriginalDocumentPositionList(); bool notEOF = GetNext(ref docList); Index.WordIndexReader wordIndexReader = wifq.WordIndex; Docid2Long last = new Docid2Long(); last.DocId = -1; int relCount = 0; while (notEOF) { //Process untokenized conditions. //If is not matched, get the next one. if (needFilterUntokenizedConditions) { int docId = docList.DocumentId; drp->DocId = docId; drp->PayloadData = dBProvider.GetPayloadDataWithShareLock(docId); if (!ParseWhere.GetComparisionExpressionValue(dBProvider, drp, untokenizedTree)) { notEOF = GetNext(ref docList); continue; } } //Process deleted records if (haveRecordsDeleted) { if (curDelIndex < delDocs.Length) { //If docid deleted, get next if (docList.DocumentId == curDelDocid) { notEOF = GetNext(ref docList); continue; } else if (docList.DocumentId > curDelDocid) { while (curDelIndex < delDocs.Length && curDelDocid < docList.DocumentId) { curDelIndex++; if (curDelIndex >= delDocs.Length) { haveRecordsDeleted = false; break; } curDelDocid = delDocs[curDelIndex]; } if (curDelIndex < delDocs.Length) { if (docList.DocumentId == curDelDocid) { notEOF = GetNext(ref docList); continue; } } } } } if (needGroupBy) { if (groupByCount < groupByLen) { if (groupByIndex >= groupByStep) { groupByIndex = 0; } if (groupByIndex == 0) { docIdRank.AddToGroupByCollection(docList.DocumentId); groupByCount++; } groupByIndex++; } } relCount++; Docid2Long cur = new Docid2Long(); if (rows >= top) { long score = 1; if (orderByIncludingScore) { int wordCount = docList.CountAndWordCount / 8; //one word, score = count score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)wordCount * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); } cur.DocId = docList.DocumentId; cur.Rank = docList.TotalWordsInThisDocument; Docid2Long.Generate(ref cur, dBProvider, orderByFields, score); if (comparer.Compare(last, cur) > 0) { priorQueue.Add(cur); last = priorQueue.Last; } } else { long score = 1; if (orderByIncludingScore) { int wordCount = docList.CountAndWordCount / 8; //one word, score = count score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)wordCount * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); } if (score < 0) { //Overflow score = long.MaxValue - 4000000; } cur.DocId = docList.DocumentId; cur.Rank = docList.TotalWordsInThisDocument; Docid2Long.Generate(ref cur, dBProvider, orderByFields, score); priorQueue.Add(cur); rows++; if (rows == top) { last = priorQueue.Last; } } notEOF = GetNext(ref docList); } docIdRank.RelTotalCount = relCount; foreach (Docid2Long docid2Long in priorQueue.ToArray()) { long score = comparer.GetScore(docid2Long); //use Rank store TotalWordsInThisDocument if (score < 0) { //Overflow score = long.MaxValue - 4000000; } docIdRank.Add(docid2Long.DocId, new DocumentResult(docid2Long.DocId, score)); } } finally { } docIdRank.Sorted = true; } } finally { Argument.DBProvider.SharedPayloadProvider.LeavePayloadShareLock(); } }