internal void Optimize(OptimizationOption option)
{
if (_IndexMerge != null)
{
_IndexMerge.Optimize(option);
}
}
private void MergeThread()
{
while (true)
{
OptimizationOption option;
lock (this)
{
if (_Closed)
{
_Thread = null;
return;
}
option = _Option;
_Option = OptimizationOption.Idle;
if (option == OptimizationOption.Idle)
{
_Thread = null;
return;
}
}
DoMerge(option);
}
}
private void DoOptimize(OptimizationOption optimization, bool showProgress)
{
_DBProvider.Optimize(optimization);
int times = 0;
while (times++ < 1800) //Time out is 3600 s
{
System.Threading.Thread.Sleep(2000);
double progress = GetMergeRate();
if (progress >= 100)
{
break;
}
if (showProgress)
{
_TableSync.SetProgress(90 + progress * 10 / 100);
}
System.Threading.Thread.Sleep(1000);
}
}
public SynchronizeCanUpdate(TableSynchronize tableSync, DBProvider dbProvider, int step,
OptimizationOption option, bool fastestMode, Service.SyncFlags flags)
{
_TableSync = tableSync;
_DBProvider = dbProvider;
_Step = step;
_OptimizeOption = option;
_FastestMode = fastestMode;
_Flags = flags;
}
public OptimizerResult Optimize(int maxIterations, OptimizationOption optimizationOption, string mlcId)
{
var local = this;
var retVal = X.Instance.CurrentContext.GetValue(sc =>
{
return(new OptimizerResult(local._client.Optimize(maxIterations, optimizationOption, mlcId)));
});
return(retVal);
}
internal static dynamic ConstructOptimizationOptionsVMAT(OptimizationOption startOption, string mlcId)
{
if (ConstructOptimizationOptionsVMATFunc0 == null)
{
throw new NullReferenceException(
"The function ConstructOptimizationOptionsVMATFunc0 has not been initialized. Try calling FacadeInitializer.Initialize() in ESAPIX.VMS before calling this method.");
}
return(XContext.Instance.CurrentContext.GetValue(sc =>
{
return ConstructOptimizationOptionsVMATFunc0(startOption, mlcId);
}));
}
internal static dynamic ConstructOptimizationOptionsIMRT(int maxIterations, OptimizationOption initialState,
OptimizationConvergenceOption convergenceOption, string mlcId)
{
if (ConstructOptimizationOptionsIMRTFunc2 == null)
{
throw new NullReferenceException(
"The function ConstructOptimizationOptionsIMRTFunc2 has not been initialized. Try calling FacadeInitializer.Initialize() in ESAPIX.VMS before calling this method.");
}
return(XContext.Instance.CurrentContext.GetValue(sc =>
{
return ConstructOptimizationOptionsIMRTFunc2(maxIterations, initialState, convergenceOption, mlcId);
}));
}
/// <summary>
/// Do synchronize
/// </summary>
/// <param name="step"></param>
/// <param name="option">optimize option</param>
/// <param name="fastestMode">if fastest mode. read data from db without log</param>
/// <param name="flags">flags</param>
public void Synchronize(int step, OptimizationOption option, bool fastestMode, SyncFlags flags)
{
if (SyncThread != null)
{
return;
}
if (!_Table.TableSynchronization && (flags & SyncFlags.Rebuild) == 0)
{
throw new DataException("Can't do synchronization. You must set the TableSynchronization to true!");
}
if (!_Table.IndexOnly && (flags & SyncFlags.Rebuild) == 0)
{
throw new DataException("Can't do synchronization in non-indexonly mode");
}
if (SyncThread != null)
{
Hubble.Core.Global.Report.WriteAppLog(string.Format("Table:{0} is synchronizing now.", _Table.Name));
return;
}
SetException(null);
_Step = step;
if (_Step <= 0)
{
_Step = 1;
}
else if (_Step > MaxStep)
{
_Step = MaxStep;
}
_OptimizeOption = option;
_FastestMode = fastestMode;
_Flags = flags;
SetProgress(0);
SyncThread = new Thread(DoSynchronize);
SyncThread.IsBackground = true;
SyncThread.Start();
}
public void Optimize(OptimizationOption option)
{
lock (this)
{
if (_Closed)
{
return;
}
_Option = option;
if (_Thread == null)
{
_Thread = new Thread(new ThreadStart(MergeThread));
_Thread.IsBackground = true;
_Thread.Start();
}
}
}
private bool DoMerge(OptimizationOption option)
{
lock (this)
{
_CanClose = true;
}
while (!_IndexFileProxy.CanMerge)
{
if (_Closed)
{
_Thread = null;
return(false);
}
System.Threading.Thread.Sleep(10);
}
Dictionary <int, System.IO.FileStream> indexSrcFileDict = new Dictionary <int, System.IO.FileStream>();
try
{
IndexFileProxy.MergeInfos mergeInfos = _IndexFileProxy.GetMergeInfos(option);
if (mergeInfos == null)
{
return(false);
}
//Console.WriteLine(string.Format("Begin={0} End={1} Serial={2}",
// mergeInfos.BeginSerial, mergeInfos.EndSerial, mergeInfos.MergedSerial));
IndexFileProxy.MergeAck mergeAck = new IndexFileProxy.MergeAck(mergeInfos.BeginSerial,
mergeInfos.EndSerial, mergeInfos.MergeHeadFileName, mergeInfos.MergeIndexFileName,
mergeInfos.MergedSerial);
//mergeInfos.MergedWordFilePostionList.Sort();
string optimizeDir = IndexDir + @"Optimize\";
if (!System.IO.Directory.Exists(optimizeDir))
{
System.IO.Directory.CreateDirectory(optimizeDir);
}
string optimizeHeadFile = optimizeDir + mergeInfos.MergeHeadFileName;
string optimizeIndexFile = optimizeDir + mergeInfos.MergeIndexFileName;
using (DDXFile headFS = new DDXFile(optimizeHeadFile, DDXFile.Mode.Write))
{
using (System.IO.FileStream indexFS = new System.IO.FileStream(optimizeIndexFile, System.IO.FileMode.Create,
System.IO.FileAccess.ReadWrite))
{
System.IO.MemoryStream m = new System.IO.MemoryStream();
_IndexFileProxy.MergeProgress = 0;
//int count = mergeInfos.Count;
long totalFileLength = mergeInfos.TotalFileLength;
int wplIndex = 0;
foreach (IndexFileProxy.MergedWordFilePostionList wpl in mergeInfos.MergedWordFilePostionList)
{
wplIndex++;
if ((wplIndex % 1000) == 0)
{
_IndexFileProxy.MergeProgress = (double)mergeInfos.FileLengthFinished / (double)totalFileLength;
}
if (_IndexFileProxy.MergeProgress > 1)
{
//Because of the count is approximate number
_IndexFileProxy.MergeProgress = 1;
}
List <Entity.MergeStream> mergeStreamList = new List <Hubble.Core.Entity.MergeStream>();
foreach (IndexFile.FilePosition fp in wpl.FilePositionList.FPList)
{
lock (this)
{
if (_Closed)
{
_Thread = null;
return(false);
}
}
if (fp.Length <= 0)
{
continue;
}
if (!indexSrcFileDict.ContainsKey(fp.Serial))
{
indexSrcFileDict.Add(fp.Serial,
new System.IO.FileStream(IndexDir + _IndexFileProxy.GetIndexFileName(fp.Serial),
System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read));
}
System.IO.FileStream fs = indexSrcFileDict[fp.Serial];
mergeStreamList.Add(new Hubble.Core.Entity.MergeStream(fs, fp.Length));
fs.Position = fp.Position;
}
if (mergeStreamList.Count > 0)
{
long position = indexFS.Position;
Entity.DocumentPositionList.Merge(mergeStreamList, indexFS);
if (headFS.Add(wpl.Word, position, indexFS.Position - position))
{
//IndexWriter.WriteHeadFile(headFS, wpl.Word, position, indexFS.Position - position);
mergeAck.MergeFilePositionList.Add(new IndexFileProxy.MergeAck.MergeFilePosition(
new IndexFile.FilePosition(mergeInfos.MergedSerial, position, (int)(indexFS.Position - position)),
wpl.Word));
}
}
}
_IndexFileProxy.MergeProgress = 1;
mergeInfos.Close();
}
}
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
lock (this)
{
if (_Closed)
{
_Thread = null;
_CanClose = true;
return(false);
}
_CanClose = false;
}
_IndexFileProxy.DoMergeAck(mergeAck);
mergeAck = null;
GC.Collect();
GC.Collect();
lock (this)
{
_CanClose = true;
}
return(true);
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
finally
{
try
{
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge close file fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
}
return(false);
}
private bool DoMerge08(OptimizationOption option)
{
lock (this)
{
_CanClose = true;
}
while (!_IndexFileProxy.CanMerge)
{
if (_Closed)
{
_Thread = null;
return(false);
}
System.Threading.Thread.Sleep(10);
}
Dictionary <int, System.IO.FileStream> indexSrcFileDict = new Dictionary <int, System.IO.FileStream>();
try
{
IndexFileProxy.MergeInfos mergeInfos = _IndexFileProxy.GetMergeInfos(option);
if (mergeInfos == null)
{
return(false);
}
IndexFileProxy.MergeAck mergeAck = new IndexFileProxy.MergeAck(mergeInfos.BeginSerial,
mergeInfos.EndSerial, mergeInfos.MergeHeadFileName, mergeInfos.MergeIndexFileName,
mergeInfos.MergedSerial);
//mergeInfos.MergedWordFilePostionList.Sort();
string optimizeDir = IndexDir + @"Optimize\";
if (!System.IO.Directory.Exists(optimizeDir))
{
System.IO.Directory.CreateDirectory(optimizeDir);
}
string optimizeHeadFile = optimizeDir + mergeInfos.MergeHeadFileName;
string optimizeIndexFile = optimizeDir + mergeInfos.MergeIndexFileName;
#if DEBUG
long TestLen = 0;
int oneFileCount = 0;
int moreFileCount = 0;
int maxFileCount = 0;
Console.WriteLine(string.Format("File ={0} Begin={1} End={2}", System.IO.Path.GetFileName(mergeInfos.MergeHeadFileName),
mergeInfos.BeginSerial, mergeInfos.EndSerial));
#endif
using (System.IO.FileStream headFS = new System.IO.FileStream(optimizeHeadFile, System.IO.FileMode.Create,
System.IO.FileAccess.ReadWrite))
{
using (System.IO.FileStream indexFS = new System.IO.FileStream(optimizeIndexFile, System.IO.FileMode.Create,
System.IO.FileAccess.ReadWrite))
{
System.IO.MemoryStream m = new System.IO.MemoryStream();
_IndexFileProxy.MergeProgress = 0;
int count = mergeInfos.Count;
int wplIndex = 0;
foreach (IndexFileProxy.MergedWordFilePostionList wpl in mergeInfos.MergedWordFilePostionList)
{
wplIndex++;
_IndexFileProxy.MergeProgress = (double)wplIndex / (double)count;
List <Entity.MergeStream> mergeStreamList = new List <Hubble.Core.Entity.MergeStream>();
foreach (IndexFile.FilePosition fp in wpl.FilePositionList.FPList)
{
lock (this)
{
if (_Closed)
{
_Thread = null;
return(false);
}
}
if (fp.Length <= 0)
{
continue;
}
if (!indexSrcFileDict.ContainsKey(fp.Serial))
{
indexSrcFileDict.Add(fp.Serial,
new System.IO.FileStream(IndexDir + _IndexFileProxy.GetIndexFileName(fp.Serial),
System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read));
}
System.IO.FileStream fs = indexSrcFileDict[fp.Serial];
mergeStreamList.Add(new Hubble.Core.Entity.MergeStream(fs, fp.Length));
fs.Position = fp.Position;
}
if (mergeStreamList.Count > 0)
{
#if DEBUG
if (mergeStreamList.Count == 1)
{
oneFileCount++;
}
else
{
moreFileCount++;
}
if (maxFileCount < mergeStreamList.Count)
{
maxFileCount = mergeStreamList.Count;
}
#endif
long position = indexFS.Position;
Entity.DocumentPositionList.Merge(mergeStreamList, indexFS);
IndexWriter.WriteHeadFile(headFS, wpl.Word, position, indexFS.Position - position);
#if DEBUG
TestLen += indexFS.Position - position;
#endif
mergeAck.MergeFilePositionList.Add(new IndexFileProxy.MergeAck.MergeFilePosition(
new IndexFile.FilePosition(mergeInfos.MergedSerial, position, (int)(indexFS.Position - position)),
wpl.Word));
}
}
}
}
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
#if DEBUG
Console.WriteLine(string.Format("{0} len={1}", fs.Name, fs.Length));
#endif
fs.Close();
}
#if DEBUG
Console.WriteLine(string.Format("Merge Len = {0} one={1} more={2} max={3}",
TestLen, oneFileCount, moreFileCount, maxFileCount));
#endif
lock (this)
{
if (_Closed)
{
_Thread = null;
_CanClose = true;
return(false);
}
_CanClose = false;
}
_IndexFileProxy.DoMergeAck(mergeAck);
mergeAck = null;
GC.Collect();
GC.Collect();
lock (this)
{
_CanClose = true;
}
return(true);
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
finally
{
try
{
foreach (System.IO.FileStream fs in indexSrcFileDict.Values)
{
fs.Close();
}
}
catch (Exception e)
{
Global.Report.WriteErrorLog(string.Format("DoMerge close file fail. err:{0} stack:{1}",
e.Message, e.StackTrace));
}
}
return(false);
}
