/// <summary> /// Collect forcedly /// </summary> public void Collect() { _IndexWriter.Close(); string ddxFile = Path.AppendDivision(_Path, '\\') + GetDDXFileName(_MaxSerial); string iFile = Path.AppendDivision(_Path, '\\') + GetIndexFileName(_MaxSerial); DDXFile iDDXFile = new DDXFile(ddxFile, DDXFile.Mode.Read); IDXFile idxfile = new IDXFile(iFile, IDXFile.Mode.Read); IndexFileInfo ifi = new IndexFileInfo(_MaxSerial, File.GetFileLength(_IndexWriter.IndexFilePath), iDDXFile, idxfile); _IndexFileList.Add(ifi); SetRamIndex(ifi, _CachedType, _MinLoadSize); _MaxSerial++; _IndexWriter = new IndexWriter(_MaxSerial, _Path, System.IO.Path.GetFileNameWithoutExtension(_FieldName), _IndexMode); }
public IndexFileInfo(int serial, FileSizeType size, DDXFile ddxFile, IDXFile idxFile) { Serial = serial; Size = size; DDXFile = ddxFile; IDXFile = idxFile; }
internal System.IO.MemoryStream GetIndexBuf(int serial, long position, long length) { IDXFile idxFile = GetIDXFile(serial); if (idxFile == null) { throw new Hubble.Core.Store.StoreException(string.Format("GetIndex Buf fail! serial={0} does not exist", serial)); } return(idxFile.GetIndexBuf(position, length)); }
/// <summary> /// Only get the word step doc index. /// </summary> /// <param name="word"></param> /// <param name="filePositionList"></param> /// <param name="totalDocs"></param> /// <param name="dbProvider"></param> /// <param name="maxReturnCount"></param> /// <returns></returns> internal Hubble.Core.Index.WordIndexReader GetWordIndexWithWordStepDocIndex(string word, WordFilePositionList filePositionList, int totalDocs, Data.DBProvider dbProvider, int maxReturnCount) { List <WordStepDocIndex.IndexFileInfo> indexFileInfoList = new List <WordStepDocIndex.IndexFileInfo>(); //StepDocIndex of all index files bool simple = _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple; long wordCountSum = 0; foreach (FilePosition filePosition in filePositionList.FPList) { IDXFile idxFile = GetIDXFile(filePosition.Serial); if (idxFile == null) { continue; } int count; long indexPosition; List <Hubble.Core.Entity.DocumentPosition> stepDocIndex = idxFile.GetStepDocIndex(filePosition.Position, filePosition.Length, out count, out indexPosition); indexFileInfoList.Add(new WordStepDocIndex.IndexFileInfo(stepDocIndex, new FilePosition(filePosition.Serial, indexPosition, (int)(filePosition.Length - (indexPosition - filePosition.Position) - sizeof(int))), //Subtract last 4 bytes for last doc id and Subtract stepdocindex length count)); wordCountSum += count; } int relDocCount = (int)wordCountSum; //if (maxReturnCount >= 0) //{ // if (wordCountSum > maxReturnCount) // { // relDocCount = maxReturnCount; // } //} WordStepDocIndex wordStepDocIndex = new WordStepDocIndex(indexFileInfoList, wordCountSum, relDocCount, simple); return(new WordIndexReader(word, wordStepDocIndex, totalDocs, dbProvider, _IndexFileProxy, maxReturnCount)); }
public IndexWriter(int serial, string path, string fieldName, Hubble.Core.Data.Field.IndexMode indexMode) { _IndexMode = indexMode; _HeadFilePath = Path.AppendDivision(path, '\\') + string.Format("{0:D7}{1}.hdx", serial, fieldName); _DDXFilePath = Path.AppendDivision(path, '\\') + string.Format("{0:D7}{1}.ddx", serial, fieldName); _IndexFilePath = Path.AppendDivision(path, '\\') + string.Format("{0:D7}{1}.idx", serial, fieldName); //_HeadFile = new System.IO.FileStream(_HeadFilePath, System.IO.FileMode.Create, // System.IO.FileAccess.Write); _DDXFile = new DDXFile(_DDXFilePath, DDXFile.Mode.Write); //_IndexFile = new System.IO.FileStream(_IndexFilePath, System.IO.FileMode.Create, // System.IO.FileAccess.Write); _IDXFile = new IDXFile(_IndexFilePath, IDXFile.Mode.Write); }
public void AfterMerge(int beginSerial, int endSerial, int mergedSerial) { int i = 0; bool fst = true; while (i < IndexFileList.Count) { if (IndexFileList[i].Serial >= beginSerial && IndexFileList[i].Serial <= endSerial) { if (fst) { string ddxFile = Path.AppendDivision(_Path, '\\') + GetDDXFileName(mergedSerial); DDXFile iDDXFile = new DDXFile(ddxFile, DDXFile.Mode.Read); string iFile = Path.AppendDivision(_Path, '\\') + GetIndexFileName(mergedSerial); IDXFile idxfile = new IDXFile(iFile, IDXFile.Mode.Read); IndexFileInfo ifi = new IndexFile.IndexFileInfo(mergedSerial, File.GetFileLength(IndexDir + GetIndexFileName(mergedSerial)), iDDXFile, idxfile); IndexFileList[i] = ifi; SetRamIndex(ifi, _CachedType, _MinLoadSize); fst = false; i++; } else { IndexFileList.RemoveAt(i); } } else { i++; } } //If IndexWriter is writting currently, don't change _MaxSerial //No longer insert after optimize if (IndexFileList[IndexFileList.Count - 1].Serial == mergedSerial && _WordFilePositionList.Count == 0) { _IndexWriter.Close(); if (System.IO.File.Exists(_IndexWriter.DDXFilePath)) { System.IO.File.Delete(_IndexWriter.DDXFilePath); } if (System.IO.File.Exists(_IndexWriter.HeadFilePath)) { System.IO.File.Delete(_IndexWriter.HeadFilePath); } if (System.IO.File.Exists(_IndexWriter.IndexFilePath)) { System.IO.File.Delete(_IndexWriter.IndexFilePath); } _MaxSerial = mergedSerial + 1; _IndexWriter = new IndexWriter(_MaxSerial, _Path, System.IO.Path.GetFileNameWithoutExtension(_FieldName), _IndexMode); } }
internal Hubble.Core.Index.WordIndexReader GetWordIndex(string word, WordFilePositionList filePositionList, int totalDocs, Data.DBProvider dbProvider, int maxReturnCount) { WordDocumentsList docList = new WordDocumentsList(); bool simple = _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple; if (maxReturnCount < 0) { foreach (FilePosition filePosition in filePositionList.FPList) { IDXFile idxFile = GetIDXFile(filePosition.Serial); if (idxFile == null) { continue; } WordDocumentsList wdl = idxFile.GetDocList(filePosition.Position, filePosition.Length, -1, simple); if (filePositionList.Count == 1) { docList = wdl; } else { docList.AddRange(wdl); docList.WordCountSum += wdl.WordCountSum; } } } else { int remain = maxReturnCount; foreach (FilePosition filePosition in filePositionList.FPList) { IDXFile idxFile = GetIDXFile(filePosition.Serial); if (idxFile == null) { continue; } WordDocumentsList wdl = idxFile.GetDocList(filePosition.Position, filePosition.Length, remain, simple); if (filePositionList.Count == 1) { docList = wdl; } else { docList.AddRange(wdl); docList.WordCountSum += wdl.WordCountSum; docList.RelDocCount += wdl.RelDocCount; } remain -= wdl.Count; } } return(new WordIndexReader(word, docList, totalDocs, dbProvider)); }
private void LoadIndexFiles(bool createNew) { if (!createNew) { TransferHeadFiles(); } string[] files = System.IO.Directory.GetFiles(_Path, "???????" + FieldName + ".ddx"); foreach (string file in files) { try { string fileName = System.IO.Path.GetFileName(file); int serial = int.Parse(fileName.Substring(0, 7)); string hFile = Path.AppendDivision(_Path, '\\') + GetHeadFileName(serial); string iFile = Path.AppendDivision(_Path, '\\') + GetIndexFileName(serial); string ddxFile = Path.AppendDivision(_Path, '\\') + GetDDXFileName(serial); if (!System.IO.File.Exists(iFile)) { if (System.IO.File.Exists(hFile)) { System.IO.File.Delete(hFile); } if (System.IO.File.Exists(ddxFile)) { System.IO.File.Delete(ddxFile); } } else if (File.GetFileLength(ddxFile) == 0 || File.GetFileLength(iFile) == 0) { try { if (System.IO.File.Exists(hFile)) { System.IO.File.Delete(hFile); } if (System.IO.File.Exists(iFile)) { System.IO.File.Delete(iFile); } if (System.IO.File.Exists(ddxFile)) { System.IO.File.Delete(ddxFile); } } catch (Exception e) { } } else { if (createNew) { try { if (System.IO.File.Exists(hFile)) { System.IO.File.Delete(hFile); } if (System.IO.File.Exists(iFile)) { System.IO.File.Delete(iFile); } if (System.IO.File.Exists(ddxFile)) { System.IO.File.Delete(ddxFile); } } catch (Exception e) { } } else { DDXFile iDDXFile = new DDXFile(ddxFile, DDXFile.Mode.Read); IDXFile idxfile = new IDXFile(iFile, IDXFile.Mode.Read); IndexFileInfo ifi = new IndexFileInfo(serial, File.GetFileLength(iFile), iDDXFile, idxfile); _IndexFileList.Add(ifi); SetRamIndex(ifi, _CachedType, _MinLoadSize); } } } catch (Exception e) { } } _IndexFileList.Sort(); //foreach (IndexFileInfo fi in _IndexFileList) //{ // using (IndexReader ir = new IndexReader(fi.Serial, _Path, FieldName, _IndexMode)) // { // List<WordFilePosition> wfp = ir.GetWordFilePositionList(); // IndexFileInterface.ImportWordFilePositionList(wfp); // wfp.Clear(); // wfp = null; // GC.Collect(); // GC.Collect(); // } //} if (_IndexFileList.Count == 0) { _MaxSerial = 0; } else { _MaxSerial = _IndexFileList[_IndexFileList.Count - 1].Serial + 1; } IndexFileInterface.CollectWordFilePositionList(); }