public void Test()
{
_requestReceived = new ManualResetEvent(false);
/*setup + start listening*/
var stub = new SipReceivedMessageProcessorStub(OnRequestReceived, (s,e) => { });
var listeningPoint = new IPEndPoint(TestConstants.MyIpAddress, 33333);
var f = new SipFactory();
var stp = new SmartThreadPool();
stp.Start();
var provider = new SipContextSource(listeningPoint, stp);
provider.AddListener(null);
provider.Start();
/*send a message to the listener*/
var send = new SipRequestBuilder().Build();
var requestBytes = SipFormatter.FormatMessage(send);
var udpClient = new UdpClient();
udpClient.Send(requestBytes, requestBytes.Length, listeningPoint);
_requestReceived.WaitOne();
var oc = ObjectComparer.Create();
var received = stub.Requests.First();
oc.Compare(received, send);
Assert.True(oc.Differences.Count == 0, oc.DifferencesString);
}
public void STPAndWIGStartSuspended()
{
STPStartInfo stpStartInfo = new STPStartInfo();
stpStartInfo.StartSuspended = true;
SmartThreadPool stp = new SmartThreadPool(stpStartInfo);
WIGStartInfo wigStartInfo = new WIGStartInfo();
wigStartInfo.StartSuspended = true;
IWorkItemsGroup wig = stp.CreateWorkItemsGroup(10, wigStartInfo);
wig.QueueWorkItem(new WorkItemCallback(this.DoWork));
Assert.IsFalse(wig.WaitForIdle(200));
wig.Start();
Assert.IsFalse(wig.WaitForIdle(200));
stp.Start();
Assert.IsTrue(wig.WaitForIdle(5000), "WIG is not idle");
Assert.IsTrue(stp.WaitForIdle(5000), "STP is not idle");
}
public BaseApproach(int autoCompleteAfterNChars, StandardEvalOutput evalOutput, PrefixProfile queryPrefixProfile)
{
_autoCompleteAfterNChars = autoCompleteAfterNChars;
_evalOutput = evalOutput;
_queryPrefixProfile = queryPrefixProfile;
_evalThreadPool = new SmartThreadPool(1000, 6);
_evalThreadPool.Start(); // Setup and start the threadpool
}
public IrrManager(Viewer viewer, int _id)
: base(viewer, _id)
{
ChangeWorkDirectory(Util.ModelFolder);
asset_max_threads = Reference.Viewer.Config.Source.Configs["Startup"].GetInt("asset_max_threads", 1);
asset_fetch_retry = Reference.Viewer.Config.Source.Configs["Startup"].GetInt("asset_fetch_retry", 2);
requestingThreadPool = new SmartThreadPool(120 * 1000, asset_max_threads);
requestingThreadPool.Start();
}
/// <summary>
/// Starts the tasks execution.
/// </summary>
/// <returns>If has reach the timeout false, otherwise true.</returns>
public override bool Start()
{
base.Start();
m_threadPool = new SmartThreadPool();
try
{
m_threadPool.MinThreads = MinThreads;
m_threadPool.MaxThreads = MaxThreads;
var workItemResults = new IWorkItemResult[Tasks.Count];
for (int i = 0; i < Tasks.Count; i++)
{
var t = Tasks[i];
workItemResults[i] = m_threadPool.QueueWorkItem(new WorkItemCallback(Run), t);
}
m_threadPool.Start();
// Timeout was reach?
if (!m_threadPool.WaitForIdle(Timeout.TotalMilliseconds > int.MaxValue ? int.MaxValue : Convert.ToInt32(Timeout.TotalMilliseconds)))
{
if (m_threadPool.IsShuttingdown)
{
return true;
}
else
{
m_threadPool.Cancel(true);
return false;
}
}
foreach (var wi in workItemResults)
{
Exception ex;
wi.GetResult(out ex);
if (ex != null)
{
throw ex;
}
}
return true;
}
finally
{
m_threadPool.Shutdown(true, 1000);
m_threadPool.Dispose();
IsRunning = false;
}
}
public Character Parse(Character character, bool force)
{
stopwatch.Reset();
Stopwatch watch = new Stopwatch();
watch.Start();
_current = character;
// Only parse the character once a day unless forced
if (character.LastParseDate.HasValue &&
character.LastParseDate.Value.AddDays(7) > DateTime.Now && !force)
{
return character;
}
ParseCharacterInformation(character);
//character.Achievements.Clear();
string mainAchievementPageUrl = string.Format("http://{2}.battle.net/wow/en/character/{0}/{1}/achievement", character.Server, character.Name, character.Region);
HtmlDocument doc = DownloadPage(mainAchievementPageUrl);
List<AchievedAchievement> achievements = new List<AchievedAchievement>();
ProcessPageForAchievements(doc.DocumentNode, character);
pagesToParse = new List<HtmlDocument>();
IList<string> extraPages = FindSubAchievementPages(doc.DocumentNode);
SmartThreadPool pool = new SmartThreadPool();
foreach (string pageUrl in extraPages)
{
pool.QueueWorkItem(new WorkItemCallback(ProcessPageOnNewThread), pageUrl);
}
pool.Start();
pool.WaitForIdle();
try
{
foreach (HtmlDocument page in pagesToParse)
{
ProcessPageForAchievements(page.DocumentNode, character);
}
}
catch (IndexOutOfRangeException)
{
}
character.LastParseDate = DateTime.Now;
character.CurrentPoints = character.TotalAchievementPoints;
watch.Stop();
return character;
}
public void QueueWorkItem_WhenMaxIsNull_Queues()
{
var info = new STPStartInfo
{
MaxQueueLength = null,
};
var pool = new SmartThreadPool(info);
pool.Start();
var workItem = pool.QueueWorkItem<object>(ReturnNull);
// If rejected, an exception would have been thrown instead.
Assert.IsTrue(workItem.GetResult() == null);
}
public void StartSuspended()
{
STPStartInfo stpStartInfo = new STPStartInfo();
stpStartInfo.StartSuspended = true;
SmartThreadPool stp = new SmartThreadPool(stpStartInfo);
stp.QueueWorkItem(new WorkItemCallback(this.DoWork));
Assert.IsFalse(stp.WaitForIdle(200));
stp.Start();
Assert.IsTrue(stp.WaitForIdle(200));
}
public void QueueWorkItem_WhenBiggerMaxIsSet_ThrowsExceptionWhenHit()
{
Assert.Throws<QueueRejectedException>(() =>
{
var info = new STPStartInfo
{
MaxQueueLength = 5,
MinWorkerThreads = 5,
MaxWorkerThreads = 10,
};
var pool = new SmartThreadPool(info);
pool.Start();
try
{
// Pool starts with 5 available waiters.
pool.QueueWorkItem(SleepForOneSecond); // Taken by waiter immediately. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // Taken by waiter immediately. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // Taken by waiter immediately. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // Taken by waiter immediately. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // Taken by waiter immediately. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // New thread created, takes work item. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // New thread created, takes work item. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // New thread created, takes work item. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // New thread created, takes work item. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // New thread created, takes work item. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // No waiters available. Queued.
pool.QueueWorkItem(SleepForOneSecond); // No waiters available. Queued.
pool.QueueWorkItem(SleepForOneSecond); // No waiters available. Queued.
pool.QueueWorkItem(SleepForOneSecond); // No waiters available. Queued.
pool.QueueWorkItem(SleepForOneSecond); // No waiters available. Queued.
}
catch (QueueRejectedException e)
{
throw new Exception("Caught QueueRejectedException too early: ", e);
}
// All threads are busy, and queue is at its max. Throws.
pool.QueueWorkItem(SleepForOneSecond);
});
}
public static SmartThreadPool GetUploadDataThreadPoolInstance()
{
if (uploadDataThreadPool == null)
{
lock (uploadDataThreadPoolLocker)
{
if (uploadDataThreadPool == null)
{
if (uploadDataThreadPoolStartInfo == null)
{
throw new Exception("线程池需要启动参数");
}
uploadDataThreadPool = new SmartThreadPool(uploadDataThreadPoolStartInfo);
uploadDataThreadPool.Start();
}
}
}
return uploadDataThreadPool;
}
public static SmartThreadPool GetCollectDataThreadPoolInstance()
{
if (collectDataThreadPool == null)
{
lock (collectDataThreadPoolLocker)
{
if (collectDataThreadPool == null)
{
if (collectDataThreadPoolStartInfo == null)
{
throw new Exception("线程池需要启动参数");
}
collectDataThreadPool = new SmartThreadPool(collectDataThreadPoolStartInfo);
collectDataThreadPool.Start();
}
}
}
return collectDataThreadPool;
}
private void Concurrency(
int concurrencyPerWig,
int wigsCount,
int workItemsCount)
{
Console.WriteLine(
"Testing : concurrencyPerWig = {0}, wigsCount = {1}, workItemsCount = {2}",
concurrencyPerWig,
wigsCount,
workItemsCount);
_success = true;
_concurrencyPerWig = concurrencyPerWig;
_randGen = new Random(0);
STPStartInfo stpStartInfo = new STPStartInfo();
stpStartInfo.StartSuspended = true;
SmartThreadPool stp = new SmartThreadPool(stpStartInfo);
_concurrentOps = new int[wigsCount];
IWorkItemsGroup [] wigs = new IWorkItemsGroup[wigsCount];
for(int i = 0; i < wigs.Length; ++i)
{
wigs[i] = stp.CreateWorkItemsGroup(_concurrencyPerWig);
for(int j = 0; j < workItemsCount; ++j)
{
wigs[i].QueueWorkItem(new WorkItemCallback(this.DoWork), i);
}
wigs[i].Start();
}
stp.Start();
stp.WaitForIdle();
Assert.IsTrue(_success);
stp.Shutdown();
}
// Can't run this test, StackOverflowException crashes the application and can't be catched and ignored
//[Test]
public void NotTestThreadsMaxStackSize()
{
STPStartInfo stpStartInfo = new STPStartInfo()
{
MaxStackSize = 64 * 1024,
};
SmartThreadPool stp = new SmartThreadPool(stpStartInfo);
stp.Start();
IWorkItemResult<bool> wir = stp.QueueWorkItem(() => AllocateBufferOnStack(10 * 1024));
bool result = wir.GetResult();
Assert.IsTrue(result);
wir = stp.QueueWorkItem(() => AllocateBufferOnStack(1000 * 1024));
result = wir.GetResult();
Assert.IsFalse(result);
}
public void DoWork(object [] states)
{
STPStartInfo stpStartInfo = new STPStartInfo();
stpStartInfo.StartSuspended = true;
SmartThreadPool smartThreadPool = new SmartThreadPool(stpStartInfo);
foreach(object state in states)
{
smartThreadPool.QueueWorkItem(new
WorkItemCallback(this.DoSomeWork), state);
}
// Start working on the work items in the queue
smartThreadPool.Start();
// Wait for the completion of all work items
smartThreadPool.WaitForIdle();
smartThreadPool.Shutdown();
}
public void CancelInQueueWorkItem()
{
Assert.Throws<WorkItemCancelException>(() =>
{
STPStartInfo stpStartInfo = new STPStartInfo();
stpStartInfo.StartSuspended = true;
bool hasRun = false;
SmartThreadPool stp = new SmartThreadPool(stpStartInfo);
IWorkItemResult wir = stp.QueueWorkItem(
new WorkItemInfo() {Timeout = 500},
state =>
{
hasRun = true;
return null;
});
Assert.IsFalse(wir.IsCanceled);
Thread.Sleep(2000);
Assert.IsTrue(wir.IsCanceled);
stp.Start();
stp.WaitForIdle();
Assert.IsFalse(hasRun);
try
{
wir.GetResult();
}
finally
{
stp.Shutdown();
}
});
}
public void DoWork()
{
STPStartInfo stpStartInfo = new STPStartInfo();
stpStartInfo.StartSuspended = true;
SmartThreadPool smartThreadPool = new SmartThreadPool();
smartThreadPool.QueueWorkItem(
new WorkItemCallback(this.DoSomeWork),
"Queued first",
WorkItemPriority.BelowNormal);
smartThreadPool.QueueWorkItem(
new WorkItemCallback(this.DoSomeWork),
"Queued second",
WorkItemPriority.AboveNormal);
smartThreadPool.Start();
smartThreadPool.WaitForIdle();
smartThreadPool.Shutdown();
}
public bool Sync()
{
bool retVal = true;
try
{
if (syncDirection == SynchronizeDirection.Upload)
{
if (logger != null) logger.Log("Start Synchronizer Enlisting Files");
List<string> fileList = AzureHelper.ListFiles(FqDirName, this.indexFileName, this.dataFileName); //what if this returns empty list
List<string> fileListToUpload = new List<string>(fileList);
if (logger != null) logger.Log("End Synchronizer Enlisting Files");
// todo filter list, to exclude archival stream files.
IWorkItemResult[] workItemResults = new IWorkItemResult[this.ThreadPoolSize];
int i = 0;
foreach (string file in fileList)
{
if (Path.GetFileName(file).Equals(this.indexFileName))
{
if (logger != null) logger.Log("Start Synchronizer Upload FileSimple");
retVal = (azureHelper.UploadFileToBlockBlob(indexFileName, file)) && retVal;
if (logger != null) logger.Log("End Synchronizer Upload FileSimple");
fileListToUpload.Remove(file);
}
else if (Path.GetFileName(file).Equals(this.dataFileName))
{
if (logger != null) logger.Log("Start Synchronizer Upload FileAsChunks");
chunkListHash = azureHelper.UploadFileAsChunks(file);
if (logger != null) logger.Log("End Synchronizer Upload FileAsChunks");
retVal = ((chunkListHash == null) ? false : true) && retVal;
fileListToUpload.Remove(file);
}
if (SyncFactory.FilesExcludedFromSync.Contains(Path.GetFileName(file)) || SyncFactory.FilesExcludedFromSync.Contains(Path.GetExtension(file)))
{
// AzureHelper.structuredLog("I", "Ignoring sync for file {0}" , file);
// do nothing. just ignore the file
fileListToUpload.Remove(file);
}
} //we've taken care of the index, data, and files to ignore. whatever is left are datablock files for file data streams (?)
if (logger != null) logger.Log("Start Synchronizer Upload FDSFiles");
SmartThreadPool threadPool = new SmartThreadPool();
threadPool.MaxThreads = this.ThreadPoolSize;
foreach (string file in fileListToUpload)
{
workItemResults[i++] = threadPool.QueueWorkItem(new WorkItemCallback(this.UploadFileToBlockBlob_worker), file);
}
threadPool.Start();
threadPool.WaitForIdle();
for (int j = 0; j < i; j++)
{
retVal = retVal && ((bool)workItemResults[j].Result);
}
threadPool.Shutdown();
if (logger != null) logger.Log("End Synchronizer Upload FDSFiles");
}
if (syncDirection == SynchronizeDirection.Download)
{
this.GetDataFileMetadata();// get chunkMD and populate chunkList Hash
retVal = azureHelper.DownloadBlockBlobToFile(indexFileName, FqDirName + "/" + indexFileName);// download index
}
}
catch(Exception e)
{
if (logger != null) logger.Log("Exception in Sync(): " + e.GetType() + ", " + e.Message);
retVal = false;
}
return retVal;
}
private void addThisStoryToQueueToolStripMenuItem_Click(object sender, EventArgs e)
{
SmartThreadPool pool = new SmartThreadPool();
foreach (SearchItem item in lvLastestUpdates.SelectedObjects)
{
pool.QueueWorkItem(this.AddToQueue, item);
}
pool.Start();
}
public void QueueWorkItem_WhenMaxIsSet_ThrowsExceptionWhenHit()
{
Assert.Throws<QueueRejectedException>(() =>
{
var info = new STPStartInfo
{
MaxQueueLength = 1,
MinWorkerThreads = 1,
MaxWorkerThreads = 1,
};
var pool = new SmartThreadPool(info);
pool.Start();
try
{
pool.QueueWorkItem(SleepForOneSecond); // Taken by waiter immediately. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // No waiters available, pool at max threads. Queued.
}
catch (QueueRejectedException e)
{
throw new Exception("Caught QueueRejectedException too early: ", e);
}
// No waiters available, queue is at max (1). Throws.
pool.QueueWorkItem(SleepForOneSecond);
});
}
protected void ManagerProc()
{
STPStartInfo startInfo = new STPStartInfo();
startInfo.MaxWorkerThreads = ExecutionThreads;
SmartThreadPool st = new SmartThreadPool(startInfo);
try
{
st.Start();
while (!_stop)
{
try
{
if (!st.IsIdle)
{
log.Warn("Thread pool not idle, waiting...");
}
else
{
log.Debug("Querying for ready processes");
IList<string> procs = Environment.GetKickableProcesses();
foreach (string procId in procs)
{
log.Debug("Queue <- {0}", procId);
st.QueueWorkItem(new WorkItemCallback(this.KickProcess), procId);
}
log.Debug("Enqueued {0} processes", procs.Count);
}
if (st.IsIdle)
{
_procNotifier.WaitOne(TimeSpan.FromSeconds(15.0), true);
}
else
{
//TODO: problem here - if some process takes a long time to
//kick, it will block process scheduling
//solution: we shouldn't wait for idle thread pool, but
//pump the queue all the time.
st.WaitForIdle(TimeSpan.FromHours(1.0));
}
}
catch (ThreadAbortException ex) {}
catch (ThreadInterruptedException ex) {}
catch (Exception ex)
{
log.Error("Manager thread error: {0}", ex);
DiagnosticEvent de = new DiagnosticEvent("NGEngine", ex);
de.Message = "NGEngine scheduler thread error";
MessageBus.Notify("NGengine", "Error", de, false);
if(!_stop) Thread.Sleep(30000);
}
}
}
catch (ThreadAbortException ex) {}
catch (ThreadInterruptedException ex) {}
catch (Exception ex)
{
log.Error("Manager thread error: {0}", ex);
}
finally
{
log.Info("Shutting down thread pool");
st.Shutdown(true, 10000);
log.Info("Thread pool shut down");
st.Dispose();
}
}
public void StpStartInfo_WithZeroMaxQueueLength_IsAllowed()
{
var info = new STPStartInfo
{
MaxQueueLength = 0,
};
var pool = new SmartThreadPool(info);
pool.Start();
Assert.True(0 == pool.STPStartInfo.MaxQueueLength);
}
public void SetMaxQueueLength_IncreasedFromZero_AllowsLargerQueue()
{
var info = new STPStartInfo
{
MinWorkerThreads = 1,
MaxWorkerThreads = 1,
MaxQueueLength = 0,
};
var pool = new SmartThreadPool(info);
pool.Start();
try
{
pool.QueueWorkItem(SleepForOneSecond); // Picked up by waiter.
}
catch (QueueRejectedException e)
{
throw new Exception("Caught QueueRejectedException too early: ", e);
}
try
{
pool.QueueWorkItem(SleepForOneSecond);
}
catch (QueueRejectedException)
{
// Expected
Assert.True(true);
}
pool.MaxQueueLength = 1;
// Don't wait for worker item to complete, the queue should have immediately increased its allowance.
var workItem = pool.QueueWorkItem<object>(ReturnNull);
// If rejected, an exception would have been thrown instead.
Assert.IsTrue(workItem.GetResult() == null);
}
public void SetMaxQueueLength_FromNullToZero_DisablesQueueing()
{
Assert.Throws<QueueRejectedException>(() =>
{
var info = new STPStartInfo
{
MinWorkerThreads = 1,
MaxWorkerThreads = 1,
};
var pool = new SmartThreadPool(info);
pool.Start();
try
{
pool.QueueWorkItem(SleepForOneSecond); // Picked up by waiter.
pool.QueueWorkItem(SleepForOneSecond); // Queued.
}
catch (QueueRejectedException e)
{
throw new Exception("Caught QueueRejectedException too early: ", e);
}
pool.MaxQueueLength = 0;
Thread.Sleep(2100); // Let the work items complete.
try
{
pool.QueueWorkItem(SleepForOneSecond); // Picked up by waiter.
}
catch (QueueRejectedException e)
{
throw new Exception("Caught QueueRejectedException too early: ", e);
}
pool.QueueWorkItem(SleepForOneSecond); // Rejected (max queue length is zero).
});
}
public static void Main(string[] args)
{
for (int i = 0; i < iSocketPool; i++)
{
objSocketQueue.Enqueue(new TcpClient("localhost", 5050));
Console.WriteLine("TCP Client Created: " + i);
}
objThread = new SmartThreadPool(100, 20, 5);
objThread.Start();
objThread.QueueWorkItem(new WorkItemCallback(Start));
Console.ReadLine();
objThread.Shutdown();
}
public void QueueWorkItem_WhenQueueMaxLengthZero_RejectsInsteadOfQueueing()
{
Assert.Throws<QueueRejectedException>(() =>
{
var info = new STPStartInfo
{
MaxQueueLength = 0,
MinWorkerThreads = 2,
MaxWorkerThreads = 2,
};
var pool = new SmartThreadPool(info);
pool.Start();
try
{
pool.QueueWorkItem(SleepForOneSecond); // Taken by waiter immediately. Not queued.
pool.QueueWorkItem(SleepForOneSecond); // Taken by waiter immediately. Not queued.
}
catch (QueueRejectedException e)
{
throw new Exception("Caught QueueRejectedException too early: ", e);
}
pool.QueueWorkItem(SleepForOneSecond);
});
}
public bool Sync()
{
if (syncDirection == SynchronizeDirection.Upload)
{
if (logger != null) logger.Log("Start Synchronizer Enlisting Files");
List<string> fileList = AzureHelper.ListFiles(localSource, this.indexFileName, this.dataFileName);
List<string> fileListToUpload = new List<string>(fileList);
if (logger != null) logger.Log("End Synchronizer Enlisting Files");
// todo filter list, to exclude archival stream files.
bool retVal = true;
foreach (string file in fileList)
{
if (Path.GetFileName(file).Equals(this.indexFileName))
{
if (logger != null) logger.Log("Start Synchronizer Upload FileSimple");
retVal = amazonS3Helper.UploadFileToS3Object(indexFileName, file);
if (logger != null) logger.Log("End Synchronizer Upload FileSimple");
fileListToUpload.Remove(file);
}
else if (Path.GetFileName(file).Equals(this.dataFileName))
{
if (logger != null) logger.Log("Start Synchronizer Upload FileAsChunks");
chunkListHash = amazonS3Helper.UploadFileAsChunks(file);
if (logger != null) logger.Log("End Synchronizer Upload FileAsChunks");
retVal = (chunkListHash == null) ? false : true;
fileListToUpload.Remove(file);
}
if (SyncFactory.FilesExcludedFromSync.Contains(Path.GetFileName(file)) || SyncFactory.FilesExcludedFromSync.Contains(Path.GetExtension(file)))
{
// AzureHelper.structuredLog("I", "Ignoring sync for file {0}" , file);
// do nothing. just ignore the file
fileListToUpload.Remove(file);
}
} //we've taken care of the index, data, and files to ignore. whatever is left are datablock files for file data streams (?)
if (logger != null) logger.Log("Start Synchronizer Upload FDSFiles");
SmartThreadPool threadPool = new SmartThreadPool();
threadPool.MaxThreads = this.MaxConcurrentFileSyncThreads;
foreach (string file in fileListToUpload)
{
IWorkItemResult wir1 = threadPool.QueueWorkItem(new WorkItemCallback(this.UploadFileToBlockBlob_worker), file);
}
threadPool.Start();
threadPool.WaitForIdle();
threadPool.Shutdown();
if (logger != null) logger.Log("End Synchronizer Upload FDSFiles");
}
if (syncDirection == SynchronizeDirection.Download)
{
this.GetDataFileMetadata();// get chunkMD and populate chunkList Hash
return amazonS3Helper.DownloadS3ObjectToFile(indexFileName, localSource + "/" + indexFileName);// download index
}
return true;//TODO fix this to return correct value for upload cases
}
private void UploadChunkList(ref List<ChunkInfo> chunkList_toUpload, string filePath, string blobName)
{
SmartThreadPool threadPool = new SmartThreadPool();
threadPool.MaxThreads = MaxConcurrentUploadThreads;
foreach (ChunkInfo chunk in chunkList_toUpload)
{
ChunkInfo chunkToUpload = chunk;
IWorkItemResult wir1 = threadPool.QueueWorkItem(new WorkItemCallback(this.UploadChunk_Worker), new Tuple<string, string, ChunkInfo>(filePath, blobName, chunkToUpload));
//UploadChunk(filePath, blobName, ref chunkToUpload);
}
threadPool.Start();
threadPool.WaitForIdle();
threadPool.Shutdown();
}
/// <summary>Get multiple Series at once. Download progress is provided trough DownloadProgressNotification event</summary>
/// <param name="series">List of Series ID's to download meta data for.</param>
/// <param name="parallelDownloads">Number of Parallel downloads</param>
/// <returns>Enumerable collection of Series</returns>
public static IEnumerable<Series> GetMulti(IList<int> series, int parallelDownloads)
{
ServicePointManager.DefaultConnectionLimit = parallelDownloads <= 5 ? 5 : parallelDownloads;
Logging.ConfigureLogging("FANART.TV-API", "fanart.tv-api.log.txt", Configuration.LogLevel);
if (parallelDownloads <= 0)
{
parallelDownloads = 10;
}
Logger.Info("Downloading {0} series metadata. Parallelism: {1}", series.Count, parallelDownloads);
lock (DownloadedXmlLock)
{
foreach (var seriesId in series)
{
if (!DownloadedXml.ContainsKey(seriesId))
{
DownloadedXml.Add(seriesId, null);
}
}
}
var downloadThreadPool = new SmartThreadPool(10000, parallelDownloads);
downloadThreadPool.Start();
foreach (var seriesId in series.Distinct())
{
downloadThreadPool.QueueWorkItem(new Amib.Threading.Func<int, bool, Series>(Get), seriesId, true);
}
downloadThreadPool.WaitForIdle();
lock (DownloadedXmlLock)
{
var returnValue = DownloadedXml.Where(s => series.Contains(s.Key)).ToDictionary(e => e.Key, e => e.Value).Select(s => s.Value);
var elementsToRemove = DownloadedXml.Keys.Where(series.Contains).ToList();
foreach (var elementToRemove in elementsToRemove)
{
DownloadedXml.Remove(elementToRemove);
}
return returnValue;
}
}
private static void Main(string[] args)
{
CLP.About();
if (args.Length == 0)
{
char[] delimiterChars = {' '};
if (File.Exists("tpccbench.txt"))
{
//attempt to open file with command line args in it
StreamReader re = File.OpenText("tpccbench.txt");
string input = re.ReadToEnd();
string[] cmdargs = input.Split(delimiterChars);
re.Close();
//send the command line to the parser
CLP.Cmdline(cmdargs);
}
else
{
Globals.Err = 1;
Console.WriteLine("tpccbench.txt not found, no commands on command line found.");
}
}
else
{
CLP.Cmdline(args);
}
if (Globals.Err == 1)
{
Console.WriteLine("Error detected, execution stopped.");
}
else if (Globals.Err == 2)
{
Console.WriteLine("Program Exit.");
}
else
{
//LoadGlobal.LoadGeneralGlobals();
LoadGlobals.LoadMasterConnectionString();
LoadGlobals.LoadLogFilePath();
var trd = new Thread(PLOG.WriteLog);
//trd.IsBackground = true;
trd.Start();
PLOG.Write("Running Benchmark on server: " + Globals.ServerName);
PLOG.Write("Running Benchmark on database: " + Globals.DatabaseName);
const string query = "select @@version";
//Console.WriteLine(query);
try
{
//Globals.SQLVersion =
ClientDataAccess.RunProc(Globals.StrPublisherConn, query);
}
catch
{
Console.WriteLine("Unable to connect to database, correct and retry");
PLOG.Write("Unable to connect to database, correct and retry", 1);
Thread.Sleep(1000);
Globals.Err = 1;
}
if (Globals.Err == 1)
{
Console.WriteLine("Error detected, execution stopped.");
}
else
{
if (Globals.Heartbeat == 1)
{
PLOG.Write("Heartbeat enabled.");
var heartbeat = new Thread(TpcUtils.Heartbeat) {IsBackground = true};
heartbeat.Start();
}
var stpStartInfo = new STPStartInfo();
if (!(Globals.StaggeredLoad == 0))
{
stpStartInfo.StartSuspended = false;
stpStartInfo.MaxWorkerThreads = Globals.NumClients + 1;
stpStartInfo.MinWorkerThreads = Globals.NumClients + 1;
stpStartInfo.IdleTimeout = 10*1000;
}
else
{
stpStartInfo.StartSuspended = true;
stpStartInfo.MaxWorkerThreads = Globals.NumClients + 1;
stpStartInfo.MinWorkerThreads = Globals.NumClients + 1;
stpStartInfo.IdleTimeout = 10*1000;
}
var smartThreadPool = new SmartThreadPool(stpStartInfo);
var s = new StopWatch();
int i = 0;
var clients = new Tpcc[Globals.NumClients];
PLOG.Write("Start TPC benchmark :" + DateTime.Now);
s.Start();
if (!(Globals.StaggeredLoad == 0))
{
while (i < Globals.NumClients)
{
clients[i] = new Tpcc {Clientid = i + 1};
smartThreadPool.QueueWorkItem(new WorkItemCallback(clients[i].Client));
Console.WriteLine("Client " + i + " Loaded.");
Thread.Sleep(Globals.StaggeredLoad);
i++;
}
}
else
{
while (i < Globals.NumClients)
{
clients[i] = new Tpcc {Clientid = i + 1};
smartThreadPool.QueueWorkItem(new WorkItemCallback(clients[i].Client));
i++;
}
}
PLOG.Write("Started On: " + DateTime.Now);
if (Globals.NumLoops == 0)
{
PLOG.Write("Target Run Time: " + (Globals.MaxRunTimeMin));
}
else
{
PLOG.Write("Number of Loops: " + (Globals.NumLoops));
}
PLOG.Write("number of clients to load: " + i);
PLOG.Write("Work load Mix:");
if (Globals.RawWrite == 0)
{
PLOG.Write("Percent New Orders: " + Globals.PNO);
PLOG.Write("Percent Order Status: " + (Globals.POS));
PLOG.Write("Percent Stock Level: " + (Globals.PSL));
PLOG.Write("Percent Payment: " + (Globals.PP));
PLOG.Write("Percent Delivery: " + (Globals.PD));
PLOG.Write("Total Percent: " +
((Globals.PNO) + (Globals.POS) + (Globals.PSL) + (Globals.PD) + (Globals.PP)));
}
else
{
PLOG.Write("Raw Inserts: 100");
}
PLOG.Read();
//Console.WriteLine("Threads waiting callback: " + smartThreadPool.WaitingCallbacks);
if (Globals.StaggeredLoad == 0)
{
smartThreadPool.Start();
}
// Wait for the completion of all work items
smartThreadPool.WaitForIdle();
s.Stop();
smartThreadPool.Shutdown();
if (Globals.RawWrite == 1)
{
PLOG.Write("Total number of operations: " + Globals.TotalCount);
PLOG.Write("elapsed time in seconds: " + s.GetElapsedTimeSecs());
PLOG.Write("elapsed time in minutes: " + s.GetElapsedTimeSecs()/60);
PLOG.Write("Total number of operations per minute: " +
(Globals.TotalCount/(s.GetElapsedTimeSecs()/60)));
}
else
{
PLOG.Write("Total number of new orders: " + Globals.Countno);
PLOG.Write("Total number of order status: " + Globals.Countos);
PLOG.Write("Total number of payments: " + Globals.Countp);
PLOG.Write("Total number of new diliveries: " + Globals.Countd);
PLOG.Write("Total number of new stock level: " + Globals.Countsl);
PLOG.Write("Total number of operations: " +
(Globals.Countno + Globals.Countos + Globals.Countp + Globals.Countd +
Globals.Countsl));
PLOG.Write("elapsed time in seconds: " + s.GetElapsedTimeSecs());
PLOG.Write("elapsed time in minutes: " + s.GetElapsedTimeSecs()/60);
PLOG.Write("Total number of new orders per minute: " +
(Globals.Countno/(s.GetElapsedTimeSecs()/60)));
}
PLOG.Write("end TPC benchmark :" + DateTime.Now);
PLOG.Read();
Globals.RunFlag = 1;
}
trd.Join(10000);
//Console.ReadKey();
}
}
public void Initialise(IConfigSource config)
{
m_proxyurl = config.Configs["Startup"].GetString("HttpProxy");
m_proxyexcepts = config.Configs["Startup"].GetString("HttpProxyExceptions");
m_outboundUrlFilter = new OutboundUrlFilter("Script HTTP request module", config);
int maxThreads = 15;
IConfig httpConfig = config.Configs["HttpRequestModule"];
if (httpConfig != null)
{
maxThreads = httpConfig.GetInt("MaxPoolThreads", maxThreads);
}
m_pendingRequests = new Dictionary<UUID, HttpRequestClass>();
// First instance sets this up for all sims
if (ThreadPool == null)
{
STPStartInfo startInfo = new STPStartInfo();
startInfo.IdleTimeout = 20000;
startInfo.MaxWorkerThreads = maxThreads;
startInfo.MinWorkerThreads = 1;
startInfo.ThreadPriority = ThreadPriority.BelowNormal;
startInfo.StartSuspended = true;
startInfo.ThreadPoolName = "ScriptsHttpReq";
ThreadPool = new SmartThreadPool(startInfo);
ThreadPool.Start();
}
}