/// <summary>
/// Collect forcedly
/// </summary>
public void Collect()
{
_IndexWriter.Close();
string ddxFile = Path.AppendDivision(_Path, '\\') +
GetDDXFileName(_MaxSerial);
string iFile = Path.AppendDivision(_Path, '\\') +
GetIndexFileName(_MaxSerial);
DDXFile iDDXFile = new DDXFile(ddxFile, DDXFile.Mode.Read);
IDXFile idxfile = new IDXFile(iFile, IDXFile.Mode.Read);
IndexFileInfo ifi = new IndexFileInfo(_MaxSerial, File.GetFileLength(_IndexWriter.IndexFilePath),
iDDXFile, idxfile);
_IndexFileList.Add(ifi);
SetRamIndex(ifi, _CachedType, _MinLoadSize);
_MaxSerial++;
_IndexWriter = new IndexWriter(_MaxSerial, _Path,
System.IO.Path.GetFileNameWithoutExtension(_FieldName), _IndexMode);
}
public IndexFileInfo(int serial, FileSizeType size, DDXFile ddxFile, IDXFile idxFile)
{
Serial = serial;
Size = size;
DDXFile = ddxFile;
IDXFile = idxFile;
}
private void TransferHeadFiles()
{
//Get .hdx files in the index folder
string[] files = System.IO.Directory.GetFiles(_Path, "???????" + FieldName + ".hdx");
//Transfer .hdx to .ddx file
foreach (string file in files)
{
string fileName = System.IO.Path.GetFileName(file);
int serial = int.Parse(fileName.Substring(0, 7));
string ddxFileName = Path.AppendDivision(_Path, '\\') +
GetDDXFileName(serial);
if (Hubble.Framework.IO.File.GetFileLength(file) <= 0)
{
System.IO.File.Delete(file);
continue;
}
if (System.IO.File.Exists(ddxFileName))
{
System.IO.File.Delete(ddxFileName);
}
using (System.IO.FileStream hFile = new System.IO.FileStream(file, System.IO.FileMode.Open,
System.IO.FileAccess.Read, System.IO.FileShare.Read))
{
using (DDXFile ddxFile = new DDXFile(ddxFileName, DDXFile.Mode.Write))
{
foreach (IndexFile.WordFilePosition wfp in GetWordFilePositionList(hFile, serial))
{
ddxFile.Add(wfp.Word, wfp.Position.Position, wfp.Position.Length);
}
ddxFile.Close();
}
}
string bakFile = Path.AppendDivision(_Path, '\\') + GetHeadBakFileName(serial);
if (System.IO.File.Exists(bakFile))
{
System.IO.File.Delete(bakFile);
}
System.IO.File.Move(file, bakFile);
}
}
public IndexWriter(int serial, string path, string fieldName, Hubble.Core.Data.Field.IndexMode indexMode)
{
_IndexMode = indexMode;
_HeadFilePath = Path.AppendDivision(path, '\\') +
string.Format("{0:D7}{1}.hdx", serial, fieldName);
_DDXFilePath = Path.AppendDivision(path, '\\') +
string.Format("{0:D7}{1}.ddx", serial, fieldName);
_IndexFilePath = Path.AppendDivision(path, '\\') +
string.Format("{0:D7}{1}.idx", serial, fieldName);
//_HeadFile = new System.IO.FileStream(_HeadFilePath, System.IO.FileMode.Create,
// System.IO.FileAccess.Write);
_DDXFile = new DDXFile(_DDXFilePath, DDXFile.Mode.Write);
//_IndexFile = new System.IO.FileStream(_IndexFilePath, System.IO.FileMode.Create,
// System.IO.FileAccess.Write);
_IDXFile = new IDXFile(_IndexFilePath, IDXFile.Mode.Write);
}
public void AfterMerge(int beginSerial, int endSerial, int mergedSerial)
{
int i = 0;
bool fst = true;
while (i < IndexFileList.Count)
{
if (IndexFileList[i].Serial >= beginSerial && IndexFileList[i].Serial <= endSerial)
{
if (fst)
{
string ddxFile = Path.AppendDivision(_Path, '\\') +
GetDDXFileName(mergedSerial);
DDXFile iDDXFile = new DDXFile(ddxFile, DDXFile.Mode.Read);
string iFile = Path.AppendDivision(_Path, '\\') +
GetIndexFileName(mergedSerial);
IDXFile idxfile = new IDXFile(iFile, IDXFile.Mode.Read);
IndexFileInfo ifi = new IndexFile.IndexFileInfo(mergedSerial,
File.GetFileLength(IndexDir + GetIndexFileName(mergedSerial)),
iDDXFile, idxfile);
IndexFileList[i] = ifi;
SetRamIndex(ifi, _CachedType, _MinLoadSize);
fst = false;
i++;
}
else
{
IndexFileList.RemoveAt(i);
}
}
else
{
i++;
}
}
//If IndexWriter is writting currently, don't change _MaxSerial
//No longer insert after optimize
if (IndexFileList[IndexFileList.Count - 1].Serial == mergedSerial &&
_WordFilePositionList.Count == 0)
{
_IndexWriter.Close();
if (System.IO.File.Exists(_IndexWriter.DDXFilePath))
{
System.IO.File.Delete(_IndexWriter.DDXFilePath);
}
if (System.IO.File.Exists(_IndexWriter.HeadFilePath))
{
System.IO.File.Delete(_IndexWriter.HeadFilePath);
}
if (System.IO.File.Exists(_IndexWriter.IndexFilePath))
{
System.IO.File.Delete(_IndexWriter.IndexFilePath);
}
_MaxSerial = mergedSerial + 1;
_IndexWriter = new IndexWriter(_MaxSerial, _Path,
System.IO.Path.GetFileNameWithoutExtension(_FieldName), _IndexMode);
}
}
private void LoadIndexFiles(bool createNew)
{
if (!createNew)
{
TransferHeadFiles();
}
string[] files = System.IO.Directory.GetFiles(_Path, "???????" + FieldName + ".ddx");
foreach (string file in files)
{
try
{
string fileName = System.IO.Path.GetFileName(file);
int serial = int.Parse(fileName.Substring(0, 7));
string hFile = Path.AppendDivision(_Path, '\\') +
GetHeadFileName(serial);
string iFile = Path.AppendDivision(_Path, '\\') +
GetIndexFileName(serial);
string ddxFile = Path.AppendDivision(_Path, '\\') +
GetDDXFileName(serial);
if (!System.IO.File.Exists(iFile))
{
if (System.IO.File.Exists(hFile))
{
System.IO.File.Delete(hFile);
}
if (System.IO.File.Exists(ddxFile))
{
System.IO.File.Delete(ddxFile);
}
}
else if (File.GetFileLength(ddxFile) == 0 || File.GetFileLength(iFile) == 0)
{
try
{
if (System.IO.File.Exists(hFile))
{
System.IO.File.Delete(hFile);
}
if (System.IO.File.Exists(iFile))
{
System.IO.File.Delete(iFile);
}
if (System.IO.File.Exists(ddxFile))
{
System.IO.File.Delete(ddxFile);
}
}
catch (Exception e)
{
}
}
else
{
if (createNew)
{
try
{
if (System.IO.File.Exists(hFile))
{
System.IO.File.Delete(hFile);
}
if (System.IO.File.Exists(iFile))
{
System.IO.File.Delete(iFile);
}
if (System.IO.File.Exists(ddxFile))
{
System.IO.File.Delete(ddxFile);
}
}
catch (Exception e)
{
}
}
else
{
DDXFile iDDXFile = new DDXFile(ddxFile, DDXFile.Mode.Read);
IDXFile idxfile = new IDXFile(iFile, IDXFile.Mode.Read);
IndexFileInfo ifi = new IndexFileInfo(serial, File.GetFileLength(iFile),
iDDXFile, idxfile);
_IndexFileList.Add(ifi);
SetRamIndex(ifi, _CachedType, _MinLoadSize);
}
}
}
catch (Exception e)
{
}
}
_IndexFileList.Sort();
//foreach (IndexFileInfo fi in _IndexFileList)
//{
// using (IndexReader ir = new IndexReader(fi.Serial, _Path, FieldName, _IndexMode))
// {
// List<WordFilePosition> wfp = ir.GetWordFilePositionList();
// IndexFileInterface.ImportWordFilePositionList(wfp);
// wfp.Clear();
// wfp = null;
// GC.Collect();
// GC.Collect();
// }
//}
if (_IndexFileList.Count == 0)
{
_MaxSerial = 0;
}
else
{
_MaxSerial = _IndexFileList[_IndexFileList.Count - 1].Serial + 1;
}
IndexFileInterface.CollectWordFilePositionList();
}
private bool DoMerge(OptimizationOption option)
{
lock (this)
{
_CanClose = true;
}
while (!_IndexFileProxy.CanMerge)
{
if (_Closed)
{
_Thread = null;
return(false);
}
System.Threading.Thread.Sleep(10);
}
Dictionary <int, System.IO.FileStream> indexSrcFileDict = new Dictionary <int, System.IO.FileStream>();
try
{
IndexFileProxy.MergeInfos mergeInfos = _IndexFileProxy.GetMergeInfos(option);
if (mergeInfos == null)
{
return(false);
}
//Console.WriteLine(string.Format("Begin={0} End={1} Serial={2}",
// mergeInfos.BeginSerial, mergeInfos.EndSerial, mergeInfos.MergedSerial));
IndexFileProxy.MergeAck mergeAck = new IndexFileProxy.MergeAck(mergeInfos.BeginSerial,
mergeInfos.EndSerial, mergeInfos.MergeHeadFileName, mergeInfos.MergeIndexFileName,
mergeInfos.MergedSerial);
//mergeInfos.MergedWordFilePostionList.Sort();
string optimizeDir = IndexDir + @"Optimize\";
if (!System.IO.Directory.Exists(optimizeDir))
{
System.IO.Directory.CreateDirectory(optimizeDir);
}
string optimizeHeadFile = optimizeDir + mergeInfos.MergeHeadFileName;
string optimizeIndexFile = optimizeDir + mergeInfos.MergeIndexFileName;
using (DDXFile headFS = new DDXFile(optimizeHeadFile, DDXFile.Mode.Write))
{
using (System.IO.FileStream indexFS = new System.IO.FileStream(optimizeIndexFile, System.IO.FileMode.Create,
System.IO.FileAccess.ReadWrite))
{
System.IO.MemoryStream m = new System.IO.MemoryStream();
_IndexFileProxy.MergeProgress = 0;
//int count = mergeInfos.Count;
long totalFileLength = mergeInfos.TotalFileLength;
int wplIndex = 0;
foreach (IndexFileProxy.MergedWordFilePostionList wpl in mergeInfos.MergedWordFilePostionList)
{
wplIndex++;
if ((wplIndex % 1000) == 0)
{
_IndexFileProxy.MergeProgress = (double)mergeInfos.FileLengthFinished / (double)totalFileLength;
}
if (_IndexFileProxy.MergeProgress > 1)
{
//Because of the count is approximate number
_IndexFileProxy.MergeProgress = 1;
}
List <Entity.MergeStream> mergeStreamList = new List <Hubble.Core.Entity.MergeStream>();
foreach (IndexFile.FilePosition fp in wpl.FilePositionList.FPList)
{
lock (this)
{
if (_Closed)
{
_Thread = null;
return(false);
}
}
if (fp.Length <= 0)
{
continue;
}
if (!indexSrcFileDict.ContainsKey(fp.Serial))
{
indexSrcFileDict.Add(fp.Serial,
new System.IO.FileStream(IndexDir + _IndexFileProxy.GetIndexFileName(fp.Serial),
System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read));
}
System.IO.FileStream fs = indexSrcFileDict[fp.Serial];
mergeStreamList.Add(new Hubble.Core.Entity.MergeStream(fs, fp.Length));
fs.Position = fp.Position;
}
if (mergeStreamList.Count > 0)
{
long position = indexFS.Position;
Entity.DocumentPositionList.Merge(mergeStreamList, indexFS);
if (headFS.Add(wpl.Word, position, indexFS.Position - position))
{
//IndexWriter.WriteHeadFile(headFS, wpl.Word, position, indexFS.Position - position);
mergeAck.MergeFilePositionList.Add(new IndexFileProxy.MergeAck.MergeFilePosition(
new IndexFile.FilePosition(mergeInfos.MergedSerial, position, (int)(indexFS.Position - position)),
wpl.Word));
}
}
}
_IndexFileProxy.MergeProgress = 1;
mergeInfos.Close();
}
}
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
lock (this)
{
if (_Closed)
{
_Thread = null;
_CanClose = true;
return(false);
}
_CanClose = false;
}
_IndexFileProxy.DoMergeAck(mergeAck);
mergeAck = null;
GC.Collect();
GC.Collect();
lock (this)
{
_CanClose = true;
}
return(true);
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
finally
{
try
{
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge close file fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
}
return(false);
}
