private bool DoMerge(OptimizationOption option)
{
lock (this)
{
_CanClose = true;
}
while (!_IndexFileProxy.CanMerge)
{
if (_Closed)
{
_Thread = null;
return(false);
}
System.Threading.Thread.Sleep(10);
}
Dictionary <int, System.IO.FileStream> indexSrcFileDict = new Dictionary <int, System.IO.FileStream>();
try
{
IndexFileProxy.MergeInfos mergeInfos = _IndexFileProxy.GetMergeInfos(option);
if (mergeInfos == null)
{
return(false);
}
//Console.WriteLine(string.Format("Begin={0} End={1} Serial={2}",
// mergeInfos.BeginSerial, mergeInfos.EndSerial, mergeInfos.MergedSerial));
IndexFileProxy.MergeAck mergeAck = new IndexFileProxy.MergeAck(mergeInfos.BeginSerial,
mergeInfos.EndSerial, mergeInfos.MergeHeadFileName, mergeInfos.MergeIndexFileName,
mergeInfos.MergedSerial);
//mergeInfos.MergedWordFilePostionList.Sort();
string optimizeDir = IndexDir + @"Optimize\";
if (!System.IO.Directory.Exists(optimizeDir))
{
System.IO.Directory.CreateDirectory(optimizeDir);
}
string optimizeHeadFile = optimizeDir + mergeInfos.MergeHeadFileName;
string optimizeIndexFile = optimizeDir + mergeInfos.MergeIndexFileName;
using (DDXFile headFS = new DDXFile(optimizeHeadFile, DDXFile.Mode.Write))
{
using (System.IO.FileStream indexFS = new System.IO.FileStream(optimizeIndexFile, System.IO.FileMode.Create,
System.IO.FileAccess.ReadWrite))
{
System.IO.MemoryStream m = new System.IO.MemoryStream();
_IndexFileProxy.MergeProgress = 0;
//int count = mergeInfos.Count;
long totalFileLength = mergeInfos.TotalFileLength;
int wplIndex = 0;
foreach (IndexFileProxy.MergedWordFilePostionList wpl in mergeInfos.MergedWordFilePostionList)
{
wplIndex++;
if ((wplIndex % 1000) == 0)
{
_IndexFileProxy.MergeProgress = (double)mergeInfos.FileLengthFinished / (double)totalFileLength;
}
if (_IndexFileProxy.MergeProgress > 1)
{
//Because of the count is approximate number
_IndexFileProxy.MergeProgress = 1;
}
List <Entity.MergeStream> mergeStreamList = new List <Hubble.Core.Entity.MergeStream>();
foreach (IndexFile.FilePosition fp in wpl.FilePositionList.FPList)
{
lock (this)
{
if (_Closed)
{
_Thread = null;
return(false);
}
}
if (fp.Length <= 0)
{
continue;
}
if (!indexSrcFileDict.ContainsKey(fp.Serial))
{
indexSrcFileDict.Add(fp.Serial,
new System.IO.FileStream(IndexDir + _IndexFileProxy.GetIndexFileName(fp.Serial),
System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read));
}
System.IO.FileStream fs = indexSrcFileDict[fp.Serial];
mergeStreamList.Add(new Hubble.Core.Entity.MergeStream(fs, fp.Length));
fs.Position = fp.Position;
}
if (mergeStreamList.Count > 0)
{
long position = indexFS.Position;
Entity.DocumentPositionList.Merge(mergeStreamList, indexFS);
if (headFS.Add(wpl.Word, position, indexFS.Position - position))
{
//IndexWriter.WriteHeadFile(headFS, wpl.Word, position, indexFS.Position - position);
mergeAck.MergeFilePositionList.Add(new IndexFileProxy.MergeAck.MergeFilePosition(
new IndexFile.FilePosition(mergeInfos.MergedSerial, position, (int)(indexFS.Position - position)),
wpl.Word));
}
}
}
_IndexFileProxy.MergeProgress = 1;
mergeInfos.Close();
}
}
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
lock (this)
{
if (_Closed)
{
_Thread = null;
_CanClose = true;
return(false);
}
_CanClose = false;
}
_IndexFileProxy.DoMergeAck(mergeAck);
mergeAck = null;
GC.Collect();
GC.Collect();
lock (this)
{
_CanClose = true;
}
return(true);
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
finally
{
try
{
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge close file fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
}
return(false);
}
private bool DoMerge08(OptimizationOption option)
{
lock (this)
{
_CanClose = true;
}
while (!_IndexFileProxy.CanMerge)
{
if (_Closed)
{
_Thread = null;
return(false);
}
System.Threading.Thread.Sleep(10);
}
Dictionary <int, System.IO.FileStream> indexSrcFileDict = new Dictionary <int, System.IO.FileStream>();
try
{
IndexFileProxy.MergeInfos mergeInfos = _IndexFileProxy.GetMergeInfos(option);
if (mergeInfos == null)
{
return(false);
}
IndexFileProxy.MergeAck mergeAck = new IndexFileProxy.MergeAck(mergeInfos.BeginSerial,
mergeInfos.EndSerial, mergeInfos.MergeHeadFileName, mergeInfos.MergeIndexFileName,
mergeInfos.MergedSerial);
//mergeInfos.MergedWordFilePostionList.Sort();
string optimizeDir = IndexDir + @"Optimize\";
if (!System.IO.Directory.Exists(optimizeDir))
{
System.IO.Directory.CreateDirectory(optimizeDir);
}
string optimizeHeadFile = optimizeDir + mergeInfos.MergeHeadFileName;
string optimizeIndexFile = optimizeDir + mergeInfos.MergeIndexFileName;
#if DEBUG
long TestLen = 0;
int oneFileCount = 0;
int moreFileCount = 0;
int maxFileCount = 0;
Console.WriteLine(string.Format("File ={0} Begin={1} End={2}", System.IO.Path.GetFileName(mergeInfos.MergeHeadFileName),
mergeInfos.BeginSerial, mergeInfos.EndSerial));
#endif
using (System.IO.FileStream headFS = new System.IO.FileStream(optimizeHeadFile, System.IO.FileMode.Create,
System.IO.FileAccess.ReadWrite))
{
using (System.IO.FileStream indexFS = new System.IO.FileStream(optimizeIndexFile, System.IO.FileMode.Create,
System.IO.FileAccess.ReadWrite))
{
System.IO.MemoryStream m = new System.IO.MemoryStream();
_IndexFileProxy.MergeProgress = 0;
int count = mergeInfos.Count;
int wplIndex = 0;
foreach (IndexFileProxy.MergedWordFilePostionList wpl in mergeInfos.MergedWordFilePostionList)
{
wplIndex++;
_IndexFileProxy.MergeProgress = (double)wplIndex / (double)count;
List <Entity.MergeStream> mergeStreamList = new List <Hubble.Core.Entity.MergeStream>();
foreach (IndexFile.FilePosition fp in wpl.FilePositionList.FPList)
{
lock (this)
{
if (_Closed)
{
_Thread = null;
return(false);
}
}
if (fp.Length <= 0)
{
continue;
}
if (!indexSrcFileDict.ContainsKey(fp.Serial))
{
indexSrcFileDict.Add(fp.Serial,
new System.IO.FileStream(IndexDir + _IndexFileProxy.GetIndexFileName(fp.Serial),
System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read));
}
System.IO.FileStream fs = indexSrcFileDict[fp.Serial];
mergeStreamList.Add(new Hubble.Core.Entity.MergeStream(fs, fp.Length));
fs.Position = fp.Position;
}
if (mergeStreamList.Count > 0)
{
#if DEBUG
if (mergeStreamList.Count == 1)
{
oneFileCount++;
}
else
{
moreFileCount++;
}
if (maxFileCount < mergeStreamList.Count)
{
maxFileCount = mergeStreamList.Count;
}
#endif
long position = indexFS.Position;
Entity.DocumentPositionList.Merge(mergeStreamList, indexFS);
IndexWriter.WriteHeadFile(headFS, wpl.Word, position, indexFS.Position - position);
#if DEBUG
TestLen += indexFS.Position - position;
#endif
mergeAck.MergeFilePositionList.Add(new IndexFileProxy.MergeAck.MergeFilePosition(
new IndexFile.FilePosition(mergeInfos.MergedSerial, position, (int)(indexFS.Position - position)),
wpl.Word));
}
}
}
}
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
#if DEBUG
Console.WriteLine(string.Format("{0} len={1}", fs.Name, fs.Length));
#endif
fs.Close();
}
#if DEBUG
Console.WriteLine(string.Format("Merge Len = {0} one={1} more={2} max={3}",
TestLen, oneFileCount, moreFileCount, maxFileCount));
#endif
lock (this)
{
if (_Closed)
{
_Thread = null;
_CanClose = true;
return(false);
}
_CanClose = false;
}
_IndexFileProxy.DoMergeAck(mergeAck);
mergeAck = null;
GC.Collect();
GC.Collect();
lock (this)
{
_CanClose = true;
}
return(true);
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
finally
{
try
{
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge close file fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
}
return(false);
}
