protected override void Dispose(bool disposing)
{
if (disposed)
{
return;
}
lock (this)
{
if (disposed)
{
return;
}
if (disposing)
{
threadPoolManager.Dispose();
}
// new shared cleanup logic
disposed = true;
base.Dispose(disposing);
}
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
threadPoolManager.Dispose();
}
base.Dispose(disposing);
}
/// <summary>
/// Starts the tasks execution.
/// </summary>
/// <returns>If has reach the timeout false, otherwise true.</returns>
public override bool Start()
{
base.Start();
m_threadPool = new SmartThreadPool();
try
{
m_threadPool.MinThreads = MinThreads;
m_threadPool.MaxThreads = MaxThreads;
var workItemResults = new IWorkItemResult[Tasks.Count];
for (int i = 0; i < Tasks.Count; i++)
{
var t = Tasks[i];
workItemResults[i] = m_threadPool.QueueWorkItem(new WorkItemCallback(Run), t);
}
m_threadPool.Start();
// Timeout was reach?
if (!m_threadPool.WaitForIdle(Timeout.TotalMilliseconds > int.MaxValue ? int.MaxValue : Convert.ToInt32(Timeout.TotalMilliseconds)))
{
if (m_threadPool.IsShuttingdown)
{
return true;
}
else
{
m_threadPool.Cancel(true);
return false;
}
}
foreach (var wi in workItemResults)
{
Exception ex;
wi.GetResult(out ex);
if (ex != null)
{
throw ex;
}
}
return true;
}
finally
{
m_threadPool.Shutdown(true, 1000);
m_threadPool.Dispose();
IsRunning = false;
}
}
protected void ManagerProc()
{
STPStartInfo startInfo = new STPStartInfo();
startInfo.MaxWorkerThreads = ExecutionThreads;
SmartThreadPool st = new SmartThreadPool(startInfo);
try
{
st.Start();
while (!_stop)
{
try
{
if (!st.IsIdle)
{
log.Warn("Thread pool not idle, waiting...");
}
else
{
log.Debug("Querying for ready processes");
IList<string> procs = Environment.GetKickableProcesses();
foreach (string procId in procs)
{
log.Debug("Queue <- {0}", procId);
st.QueueWorkItem(new WorkItemCallback(this.KickProcess), procId);
}
log.Debug("Enqueued {0} processes", procs.Count);
}
if (st.IsIdle)
{
_procNotifier.WaitOne(TimeSpan.FromSeconds(15.0), true);
}
else
{
//TODO: problem here - if some process takes a long time to
//kick, it will block process scheduling
//solution: we shouldn't wait for idle thread pool, but
//pump the queue all the time.
st.WaitForIdle(TimeSpan.FromHours(1.0));
}
}
catch (ThreadAbortException ex) {}
catch (ThreadInterruptedException ex) {}
catch (Exception ex)
{
log.Error("Manager thread error: {0}", ex);
DiagnosticEvent de = new DiagnosticEvent("NGEngine", ex);
de.Message = "NGEngine scheduler thread error";
MessageBus.Notify("NGengine", "Error", de, false);
if(!_stop) Thread.Sleep(30000);
}
}
}
catch (ThreadAbortException ex) {}
catch (ThreadInterruptedException ex) {}
catch (Exception ex)
{
log.Error("Manager thread error: {0}", ex);
}
finally
{
log.Info("Shutting down thread pool");
st.Shutdown(true, 10000);
log.Info("Thread pool shut down");
st.Dispose();
}
}
public static List<AlignmentInfoElement> AlignPairsMultiThreaded(MPAlignerConfiguration configuration, Dictionary<string, ProcessedTermEntry> srcTerms, Dictionary<string, ProcessedTermEntry> trgTerms, bool interlinguaDictUsed, bool interlinguaTranslitUsed, string srcLang, string trgLang, string srcFile, string trgFile, Dictionary<string, Dictionary<string, bool>> excDict, Dictionary<string, bool> srcStopWords, Dictionary<string, bool> trgStopWords)
{
if (configuration == null||configuration.langPairEntryDict==null||string.IsNullOrWhiteSpace(srcLang)||string.IsNullOrWhiteSpace(trgLang))
{
return null;
}
Log.Write ("Starting alignmet of "+ srcTerms.Count.ToString()+" "+srcLang+" and "+ trgTerms.Count.ToString()+" "+trgLang+" terms.",LogLevelType.LIMITED_OUTPUT);
int threadCount = configuration.alignmentThreads;
STPStartInfo stpStartInfo = new STPStartInfo();
stpStartInfo.IdleTimeout = 100*1000;
stpStartInfo.MaxWorkerThreads = 5*threadCount;
stpStartInfo.MinWorkerThreads = threadCount;
stpStartInfo.EnableLocalPerformanceCounters = true;
SmartThreadPool smartThreadPool = new SmartThreadPool(stpStartInfo);
string langKey = srcLang+"_"+trgLang;
MPAlignerConfigurationLangPairEntry lpeConf = new MPAlignerConfigurationLangPairEntry();
if (configuration.langPairEntryDict.ContainsKey(langKey))
{
lpeConf = configuration.langPairEntryDict[langKey];
}
else
{
lpeConf = new MPAlignerConfigurationLangPairEntry();
lpeConf.srcLang = srcLang;
lpeConf.trgLang = trgLang;
}
int counter = 0;
//threadedAlignments = new List<AlignmentInfoElement>();
List<AlignmentInfoElement> res = new List<AlignmentInfoElement>();
Dictionary<string,Dictionary<string,bool>> alignedList = new Dictionary<string, Dictionary<string, bool>>();
List<IWorkItemResult<AlignmentInfoElement>> wirList = new List<IWorkItemResult<AlignmentInfoElement>>(1000);
_configuration = configuration;
_interlinguaDictUsed=interlinguaDictUsed;
_interlinguaTranslitUsed=interlinguaTranslitUsed;
_srcFile=srcFile;
_trgFile=trgFile;
_excDict=excDict;
_srcStopWords=srcStopWords;
_trgStopWords=trgStopWords;
_lpeConf=lpeConf;
foreach(string srcTerm in srcTerms.Keys)
{
counter++;
if (counter%50==0)
{
Console.Write(".");
if (counter%1000==0)
{
Console.WriteLine(" - "+counter.ToString());
}
}
ProcessedTermEntry srcPte = srcTerms[srcTerm];
foreach(string trgTerm in trgTerms.Keys)
{
//List<Tuple<ProcessedTermEntry,ProcessedTermEntry>> unProcessed = new List<Tuple<ProcessedTermEntry, ProcessedTermEntry>>();
if (wirList.Count>=100000)
{
smartThreadPool.WaitForIdle();
for(int i=0;i<wirList.Count;i++)
{
if (wirList[i].IsCompleted && wirList[i].Exception==null)
{
AlignmentInfoElement aie = (AlignmentInfoElement)wirList[i].Result;
if (aie!=null && (!alignedList.ContainsKey(aie.alignedLowSrcStr)||!alignedList[aie.alignedLowSrcStr].ContainsKey(aie.alignedLowTrgStr)))
{
res.Add(aie);
if (!alignedList.ContainsKey(aie.alignedLowSrcStr)) alignedList.Add(aie.alignedLowSrcStr,new Dictionary<string, bool>());
if (!alignedList[aie.alignedLowSrcStr].ContainsKey(aie.alignedLowTrgStr)) alignedList[aie.alignedLowSrcStr].Add(aie.alignedLowTrgStr,true);
}
}
else if (!wirList[i].IsCompleted)
{
int times = 100;
while(!wirList[i].IsCompleted && times>0)
{
times--;
System.Threading.Thread.Sleep(100);
}
if (wirList[i].IsCompleted && wirList[i].Exception==null)
{
AlignmentInfoElement aie = (AlignmentInfoElement)wirList[i].Result;
if (aie!=null && (!alignedList.ContainsKey(aie.alignedLowSrcStr)||!alignedList[aie.alignedLowSrcStr].ContainsKey(aie.alignedLowTrgStr)))
{
res.Add(aie);
if (!alignedList.ContainsKey(aie.alignedLowSrcStr)) alignedList.Add(aie.alignedLowSrcStr,new Dictionary<string, bool>());
if (!alignedList[aie.alignedLowSrcStr].ContainsKey(aie.alignedLowTrgStr)) alignedList[aie.alignedLowSrcStr].Add(aie.alignedLowTrgStr,true);
}
}
}
}
wirList.Clear();
}
try
{
IWorkItemResult<AlignmentInfoElement> wir = smartThreadPool.QueueWorkItem(
new Amib.Threading.Func<ProcessedTermEntry, ProcessedTermEntry, AlignmentInfoElement>(AlignSingleTermPair), srcPte, trgTerms[trgTerm]);
if (wir!=null) wirList.Add(wir);
}
catch
{
Log.Write("Thread exception catched - cannot create a new thread within term alignment!", LogLevelType.WARNING);
}
//smartThreadPool
/*while(smartThreadPool.PerformanceCountersReader.WorkItemsQueued>=100)
{
System.Threading.Thread.Sleep(5);
}*/
//AlignmentInfoElement aie = AlignSingleTermPair (configuration, trgTerms[trgTerm], interlinguaDictUsed, interlinguaTranslitUsed, srcFile, trgFile, excDict, srcStopWords, trgStopWords, lpeConf, srcPte);
//if (aie!=null)
//{
//res.Add(aie);
//}
}
}
//Console.WriteLine();
if (wirList.Count>0)
{
smartThreadPool.WaitForIdle();
for(int i=0;i<wirList.Count;i++)
{
if (wirList[i].IsCompleted && wirList[i].Exception==null)
{
AlignmentInfoElement aie = (AlignmentInfoElement)wirList[i].Result;
if (aie!=null && (!alignedList.ContainsKey(aie.alignedLowSrcStr)||!alignedList[aie.alignedLowSrcStr].ContainsKey(aie.alignedLowTrgStr)))
{
res.Add(aie);
if (!alignedList.ContainsKey(aie.alignedLowSrcStr)) alignedList.Add(aie.alignedLowSrcStr,new Dictionary<string, bool>());
if (!alignedList[aie.alignedLowSrcStr].ContainsKey(aie.alignedLowTrgStr)) alignedList[aie.alignedLowSrcStr].Add(aie.alignedLowTrgStr,true);
}
}
else if (!wirList[i].IsCompleted)
{
int times = 100;
while(!wirList[i].IsCompleted && times>0)
{
times--;
System.Threading.Thread.Sleep(100);
}
if (wirList[i].IsCompleted && wirList[i].Exception==null)
{
AlignmentInfoElement aie = (AlignmentInfoElement)wirList[i].Result;
if (aie!=null && (!alignedList.ContainsKey(aie.alignedLowSrcStr)||!alignedList[aie.alignedLowSrcStr].ContainsKey(aie.alignedLowTrgStr)))
{
res.Add(aie);
if (!alignedList.ContainsKey(aie.alignedLowSrcStr)) alignedList.Add(aie.alignedLowSrcStr,new Dictionary<string, bool>());
if (!alignedList[aie.alignedLowSrcStr].ContainsKey(aie.alignedLowTrgStr)) alignedList[aie.alignedLowSrcStr].Add(aie.alignedLowTrgStr,true);
}
}
}
}
wirList.Clear();
}
try{
smartThreadPool.Shutdown(true,100);
smartThreadPool.Dispose();
smartThreadPool = null;
GC.Collect();
GC.WaitForPendingFinalizers();
}
catch
{
try
{
smartThreadPool.Shutdown(true,100);
smartThreadPool.Dispose();
smartThreadPool = null;
GC.Collect();
GC.WaitForPendingFinalizers();
}
catch
{
GC.Collect();
GC.WaitForPendingFinalizers();
}
}
Log.Write ("Alignmet finished - "+ res.Count.ToString()+" term pairs aligned over the alignment threshold " +lpeConf.finalAlignmentThr.ToString()+".\n",LogLevelType.LIMITED_OUTPUT);
return res;
}