private void TransferHeadFiles()
{
//Get .hdx files in the index folder
string[] files = System.IO.Directory.GetFiles(_Path, "???????" + FieldName + ".hdx");
//Transfer .hdx to .ddx file
foreach (string file in files)
{
string fileName = System.IO.Path.GetFileName(file);
int serial = int.Parse(fileName.Substring(0, 7));
string ddxFileName = Path.AppendDivision(_Path, '\\') +
GetDDXFileName(serial);
if (Hubble.Framework.IO.File.GetFileLength(file) <= 0)
{
System.IO.File.Delete(file);
continue;
}
if (System.IO.File.Exists(ddxFileName))
{
System.IO.File.Delete(ddxFileName);
}
using (System.IO.FileStream hFile = new System.IO.FileStream(file, System.IO.FileMode.Open,
System.IO.FileAccess.Read, System.IO.FileShare.Read))
{
using (DDXFile ddxFile = new DDXFile(ddxFileName, DDXFile.Mode.Write))
{
foreach (IndexFile.WordFilePosition wfp in GetWordFilePositionList(hFile, serial))
{
ddxFile.Add(wfp.Word, wfp.Position.Position, wfp.Position.Length);
}
ddxFile.Close();
}
}
string bakFile = Path.AppendDivision(_Path, '\\') + GetHeadBakFileName(serial);
if (System.IO.File.Exists(bakFile))
{
System.IO.File.Delete(bakFile);
}
System.IO.File.Move(file, bakFile);
}
}
public long AddWordAndDocList(string word, DocumentPositionList first, int docsCount, IEnumerable <Entity.DocumentPositionList> docList, out int length)
{
long position = _IDXFile.AddWordAndDocList(word, _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple,
first, docsCount, docList, out length);
//long position = _IndexFile.Position;
//bool simple = _IndexMode == Hubble.Core.Data.Field.IndexMode.Simple;
//Entity.DocumentPositionList.Serialize(first, docsCount, docList, _IndexFile, simple);
//length = (int)(_IndexFile.Position - position);
//WriteHeadFile(_HeadFile, word, position, length);
_DDXFile.Add(word, position, length);
return(position);
}
private bool DoMerge(OptimizationOption option)
{
lock (this)
{
_CanClose = true;
}
while (!_IndexFileProxy.CanMerge)
{
if (_Closed)
{
_Thread = null;
return(false);
}
System.Threading.Thread.Sleep(10);
}
Dictionary <int, System.IO.FileStream> indexSrcFileDict = new Dictionary <int, System.IO.FileStream>();
try
{
IndexFileProxy.MergeInfos mergeInfos = _IndexFileProxy.GetMergeInfos(option);
if (mergeInfos == null)
{
return(false);
}
//Console.WriteLine(string.Format("Begin={0} End={1} Serial={2}",
// mergeInfos.BeginSerial, mergeInfos.EndSerial, mergeInfos.MergedSerial));
IndexFileProxy.MergeAck mergeAck = new IndexFileProxy.MergeAck(mergeInfos.BeginSerial,
mergeInfos.EndSerial, mergeInfos.MergeHeadFileName, mergeInfos.MergeIndexFileName,
mergeInfos.MergedSerial);
//mergeInfos.MergedWordFilePostionList.Sort();
string optimizeDir = IndexDir + @"Optimize\";
if (!System.IO.Directory.Exists(optimizeDir))
{
System.IO.Directory.CreateDirectory(optimizeDir);
}
string optimizeHeadFile = optimizeDir + mergeInfos.MergeHeadFileName;
string optimizeIndexFile = optimizeDir + mergeInfos.MergeIndexFileName;
using (DDXFile headFS = new DDXFile(optimizeHeadFile, DDXFile.Mode.Write))
{
using (System.IO.FileStream indexFS = new System.IO.FileStream(optimizeIndexFile, System.IO.FileMode.Create,
System.IO.FileAccess.ReadWrite))
{
System.IO.MemoryStream m = new System.IO.MemoryStream();
_IndexFileProxy.MergeProgress = 0;
//int count = mergeInfos.Count;
long totalFileLength = mergeInfos.TotalFileLength;
int wplIndex = 0;
foreach (IndexFileProxy.MergedWordFilePostionList wpl in mergeInfos.MergedWordFilePostionList)
{
wplIndex++;
if ((wplIndex % 1000) == 0)
{
_IndexFileProxy.MergeProgress = (double)mergeInfos.FileLengthFinished / (double)totalFileLength;
}
if (_IndexFileProxy.MergeProgress > 1)
{
//Because of the count is approximate number
_IndexFileProxy.MergeProgress = 1;
}
List <Entity.MergeStream> mergeStreamList = new List <Hubble.Core.Entity.MergeStream>();
foreach (IndexFile.FilePosition fp in wpl.FilePositionList.FPList)
{
lock (this)
{
if (_Closed)
{
_Thread = null;
return(false);
}
}
if (fp.Length <= 0)
{
continue;
}
if (!indexSrcFileDict.ContainsKey(fp.Serial))
{
indexSrcFileDict.Add(fp.Serial,
new System.IO.FileStream(IndexDir + _IndexFileProxy.GetIndexFileName(fp.Serial),
System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read));
}
System.IO.FileStream fs = indexSrcFileDict[fp.Serial];
mergeStreamList.Add(new Hubble.Core.Entity.MergeStream(fs, fp.Length));
fs.Position = fp.Position;
}
if (mergeStreamList.Count > 0)
{
long position = indexFS.Position;
Entity.DocumentPositionList.Merge(mergeStreamList, indexFS);
if (headFS.Add(wpl.Word, position, indexFS.Position - position))
{
//IndexWriter.WriteHeadFile(headFS, wpl.Word, position, indexFS.Position - position);
mergeAck.MergeFilePositionList.Add(new IndexFileProxy.MergeAck.MergeFilePosition(
new IndexFile.FilePosition(mergeInfos.MergedSerial, position, (int)(indexFS.Position - position)),
wpl.Word));
}
}
}
_IndexFileProxy.MergeProgress = 1;
mergeInfos.Close();
}
}
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
lock (this)
{
if (_Closed)
{
_Thread = null;
_CanClose = true;
return(false);
}
_CanClose = false;
}
_IndexFileProxy.DoMergeAck(mergeAck);
mergeAck = null;
GC.Collect();
GC.Collect();
lock (this)
{
_CanClose = true;
}
return(true);
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
finally
{
try
{
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge close file fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
}
return(false);
}
