public void AddWordAndDocList(string word, DocumentPositionList first, int docsCount, IEnumerable <Entity.DocumentPositionList> docList) { FileLengthType length; FilePositionType position = _IndexWriter.AddWordAndDocList(word, first, docsCount, docList, out length); _WordFilePositionList.Add(new WordFilePosition(word, _MaxSerial, position, length)); }
/// <summary> /// Get step doc index. /// </summary> /// <param name="position">first byte of the index (include step doc index)</param> /// <param name="length">index buffer length</param> /// <param name="count">count of the document records in this index</param> /// <param name="indexPostion">first byte of the index (exclude step doc index)<</param> /// <returns>step doc index. If no step doc index, return null</returns> public List <DocumentPosition> GetStepDocIndex(long position, long length, out int count, out long indexPostion) { Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport(); List <DocumentPosition> result; //indexPostion = position; _IndexFile.Seek(position, System.IO.SeekOrigin.Begin); result = DocumentPositionList.DeserializeSkipDocIndex(_IndexFile, false, out count); if (result != null) { count = DocumentPositionList.GetDocumentsCount(_IndexFile); } indexPostion = _IndexFile.Position; //The index position exclude count; performanceReport.Stop(string.Format("Read index file: len={0}, {1} results. ", _IndexFile.Position - position, count)); return(result); }
/// <summary> /// Get next document position list /// </summary> /// <returns>if end of the index, docid of result is -1</returns> public DocumentPositionList GetNext() { if (_End) { return(new DocumentPositionList(-1)); } if (_CurrentDocId < 0) { if (!NextFileIndex()) { //End of the index return(new DocumentPositionList(-1)); } } if (_CurrentIndexBuf.Position < _CurrentIndexBuf.Start + _CurrentIndexBuf.Length) { return(DocumentPositionList.GetNextDocumentPositionList(ref _CurrentDocId, _CurrentIndexBuf, _WordStepDocIndex.SimpleMode)); } else { if (!NextFileIndexBuf()) { return(new DocumentPositionList(-1)); } return(DocumentPositionList.GetNextDocumentPositionList(ref _CurrentDocId, _CurrentIndexBuf, _WordStepDocIndex.SimpleMode)); } }
internal DocIndexInfo(DocumentPositionList dpl) { this.DocumentId = dpl.DocumentId; this.FirstPosition = dpl.FirstPosition; this.Count = dpl.Count; this.TotalWordsInThisDocumentIndex = dpl.TotalWordsInThisDocumentIndex; //this.PayloadData = dbProvider.GetPayloadData(this.DocumentId); }
internal DocumentPositionList GetDocPositionList() { DocumentPositionList dpl = new DocumentPositionList(); dpl.DocumentId = this.DocumentId; dpl.FirstPosition = this.FirstPosition; dpl.Count = this.Count; dpl.SetTotalWordsInThisDocumentIndex(this.TotalWordsInThisDocumentIndex); //this.PayloadData = dbProvider.GetPayloadData(this.DocumentId); return(dpl); }
public void Reset() { _CurrentFileIndex = -1; _CurrentIndexBuf = null; _CurrentDocId = -1; _LastDocIdInCurrentStep = -1; _CurrentStepDocIndex = -1; _End = false; _NextDocPositionList = new DocumentPositionList(-1); _NextODPL.DocumentId = -1; }
/// <summary> /// Add document list of one word into inverted index file. /// </summary> /// <param name="word">word</param> /// <param name="simple">is it simple index?</param> /// <param name="first">first document position information</param> /// <param name="docsCount">document count that want to added</param> /// <param name="docList">doucment position list</param> /// <param name="length">output the length of index content</param> /// <returns>position of the first byte of this word's index in .idx file</returns> public long AddWordAndDocList(string word, bool simple, DocumentPositionList first, int docsCount, IEnumerable <Entity.DocumentPositionList> docList, out int length) { long position = _IndexFile.Position; Entity.DocumentPositionList.Serialize(first, docsCount, docList, _IndexFile, simple); length = (int)(_IndexFile.Position - position); return(position); }
public DocumentPositionList Get(int docId) { if (_End) { return(new DocumentPositionList(-1)); } if (docId < 0) { throw new StoreException(string.Format("Invalid docid = {0}", docId)); } if (_CurrentFileIndex < 0) { _NextDocPositionList = GetNext(); } if (docId < _NextDocPositionList.DocumentId) { return(new DocumentPositionList(-1)); } if (docId == _NextDocPositionList.DocumentId) { DocumentPositionList result = _NextDocPositionList; _NextDocPositionList = GetNext(); return(result); } Match(docId); //Get the matched step doc index _NextDocPositionList = GetNext(); while (_NextDocPositionList.DocumentId < docId && _NextDocPositionList.DocumentId >= 0) { _NextDocPositionList = GetNext(); } if (docId == _NextDocPositionList.DocumentId) { DocumentPositionList result = _NextDocPositionList; _NextDocPositionList = GetNext(); return(result); } else { return(new DocumentPositionList(-1)); } }
internal int Alloc() { _Current++; if (_Current >= DocPositionPool.Length) { DocumentPositionList[] tempPool = new DocumentPositionList[DocPositionPool.Length * 2]; Array.Copy(DocPositionPool, tempPool, DocPositionPool.Length); DocPositionPool = tempPool; } return(_Current); }
public long AddWordAndDocList(string word, DocumentPositionList first, int docsCount, IEnumerable <Entity.DocumentPositionList> docList, out int length) { long position = _IDXFile.AddWordAndDocList(word, _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple, first, docsCount, docList, out length); //long position = _IndexFile.Position; //bool simple = _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple; //Entity.DocumentPositionList.Serialize(first, docsCount, docList, _IndexFile, simple); //length = (int)(_IndexFile.Position - position); //WriteHeadFile(_HeadFile, word, position, length); _DDXFile.Add(word, position, length); return(position); }
/// <summary> /// Index one document into the _ListForWriter /// </summary> internal void Index() { try { if (_IndexMode == Hubble.Core.Data.Field.IndexMode.Simple) { Add(new DocumentPositionList(TempDocId, TempWordCountInThisDoc, DocumentPositionList.GetTotalWordsInDocIndex(TempTotalWordsInDoc))); //_ListForWriter.Add(new DocumentPositionList(DocId, WordCountInThisDoc, // DocumentPositionList.GetTotalWordsInDocIndex(TempTotalWordsInDoc))); } else { if (TempWordCountInThisDoc > 0) { Add(new DocumentPositionList(TempDocId, TempWordCountInThisDoc, DocumentPositionList.GetTotalWordsInDocIndex(TempTotalWordsInDoc), TempFirstPosition)); //_ListForWriter.Add(new DocumentPositionList(DocId, // WordCountInThisDoc, // DocumentPositionList.GetTotalWordsInDocIndex(TempTotalWordsInDoc), // FirstPosition)); } else { Add(new DocumentPositionList(TempDocId, TempWordCountInThisDoc, DocumentPositionList.GetTotalWordsInDocIndex(TempTotalWordsInDoc))); //_ListForWriter.Add(new DocumentPositionList(DocId, WordCountInThisDoc, // DocumentPositionList.GetTotalWordsInDocIndex(TempTotalWordsInDoc))); } } _Count++; } finally { } }
private void Add(DocumentPositionList docPositionList) { int docPositionId = _DocPositionAlloc.Alloc(); if (_First < 0) { _First = docPositionId; } _DocPositionAlloc.DocPositionPool[docPositionId] = docPositionList; if (_Cur < 0) { _Cur = _First; } else { _DocPositionAlloc.DocPositionPool[_Cur].Next = docPositionId; _Cur = docPositionId; } }
/// <summary> /// Get next original document position list /// </summary> /// <returns>if end of the index, docid of result is -1</returns> public bool GetNextOriginal(ref OriginalDocumentPositionList odpl) { if (_End) { odpl.DocumentId = -1; return(false); //return new OriginalDocumentPositionList(-1); } if (_CurrentDocId < 0) { if (!NextFileIndex()) { //End of the index odpl.DocumentId = -1; return(false); //return new OriginalDocumentPositionList(-1); } } if (_CurrentIndexBuf.Position < _CurrentIndexBuf.Start + _CurrentIndexBuf.Length) { DocumentPositionList.GetNextOriginalDocumentPositionList(ref odpl, ref _CurrentDocId, _CurrentIndexBuf, _WordStepDocIndex.SimpleMode); return(true); } else { if (!NextFileIndexBuf()) { odpl.DocumentId = -1; return(false); //return new OriginalDocumentPositionList(-1); } DocumentPositionList.GetNextOriginalDocumentPositionList(ref odpl, ref _CurrentDocId, _CurrentIndexBuf, _WordStepDocIndex.SimpleMode); return(true); } }
public Store.WordDocumentsList GetDocumentPositionList(Store.WordDocumentsList docList) { lock (this) { Store.WordDocumentsList result; bool needDel = false; for (int i = 0; i < docList.Count; i++) { if (_DeleteTbl.ContainsKey(docList[i].DocumentId)) { needDel = true; docList[i] = new DocumentPositionList(-1); } } if (needDel) { result = new Store.WordDocumentsList(); for (int i = 0; i < docList.Count; i++) { if (docList[i].DocumentId < 0) { result.Add(docList[i]); } } } else { result = docList; } return(result); } }