static void Main(string[] args)
{
MultiThreadOrchestrator orchestrator = new MultiThreadOrchestrator();
orchestrator.Orchestrate();
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Processing Finished");
}
public MultiThreadWorker(String name, ConcurrentQueue <InputDTO> wQ, ConcurrentQueue <ServiceResult> rQ)
{
threadName = name;
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Creating " + threadName);
workerQueue = wQ;
resultQueue = rQ;
}
private InputDTO getFromWorkerQ()
{
lock (workerQueue) {
if (workerQueue.Count() > 0)
{
// take() and poll(timeout) can block
// the timeout is erratic - often at least 5 to 10 times the value
// polling can cause a blocking loop that can run for up to 500ms for all workers
// by checking the Q length before trying take(), we generally
// have sufficient rows to satisfy all workers.
long t1 = CommonConstants.currentTimeMicros();
InputDTO dto;
bool gotit = workerQueue.TryDequeue(out dto);
long t2 = CommonConstants.currentTimeMicros() - t1;
if (t2 > 10)
{ // 10th of milisecond
Console.WriteLine(CommonConstants.currentTimeExtended() + ": " + threadName + " dequeue timeout=" + t2 + " micros");
}
if (gotit)
{
isProcessing = true;
return(dto);
}
else
{
return(null);
}
}
else
{
return(null);
}
}
}
public void start()
{
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Starting " + threadName);
if (t == null)
{
t = new Thread(doWork);
t.Start();
}
}
private void processResultQItem()
{
// process 1 entry in our input q
Console.WriteLine(CommonConstants.currentTimeExtended() + ": processing ResultQ len=" + getResultQLength());
ServiceResult sr = getFromResultQ();
if (sr != null) // do stuff with it
{
//Console.WriteLine(CommonConstants.currentTimeExtended() + ": res " + sr.getResult() + ", rec=" + sr.getDto().RowNbr + ", data [" + sr.getDto().Data + "]");
}
Thread.Sleep(pseudoProcessingSleepTimeValue);
}
private void performInit()
{
Console.WriteLine(CommonConstants.currentTimeExtended() + ": ##### Starting Application");
//readConfigFile(configFilePath);
// Update 'urlSuffix' value to include 'clientID'
//urlSuffix = urlSuffix + "&client=" + clientID;
// Create Service 'StopFile'
//File stopFile = new File(serviceStopFile);
//return stopFile;
}
private void closeDBConnection(bool stopFile)
{
if (stopFile)
{
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Finishing Application - 'Stop' File was found...");
}
else
{
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Finishing Application - Something called 'exit()'...");
}
if (simulateDB)
{
try
{
dbOutput.Close();
}
catch (IOException e)
{
// TODO Auto-generated catch block
Console.WriteLine(e.StackTrace);
}
}
}
public void Orchestrate()
{
performInit();
openDBConnection();
int counter = 0;
int burst = 0;
int wqLen = 0;
// ////////////////////////////////////////////
workerQueue = new ConcurrentQueue <InputDTO>();
resultQueue = new ConcurrentQueue <ServiceResult>();
workers = new List <MultiThreadWorker>(maxThreads);
for (int i = 0; i < nbrThreadsInit; i++)
{
workers.Add(new MultiThreadWorker("Worker " + i, workerQueue, resultQueue));
}
// ////////////////////////////////////////////
long currentTime = CommonConstants.currentTimeMillis();
long startTime = currentTime;
int timeDelta = 0;
int saveTimeDelta = 0;
while (!stopping)
{
// we should come through here once per second
// almost1Second is exactly that - if we construct the loop for exactly 1 second,
// we go past that value by as much as 35 ms
// recalculate it here, because we might adjust the housekeepingSleepTimeValue
almost1Second = 1000 - housekeepingSleepTimeValue + 1;
// read dbOutput requestsPerSecond rows at a time, and apportion to workers
// if the workerQueue contains rows, read less rows to ensure the queue total
// will be the configured batch size
try {
List <String> rows = performQuery(wqLen); // could set stopping = true
Console.WriteLine(CommonConstants.currentTimeExtended() + ": read " + rows.Count + " qLen = " + wqLen);
burst = rows.Count + getWorkerQLength(); // how many we are trying to process this second
if (StopFileExists(serviceStopFile))
{
stopping = true; // only set flag to allow orderly shutdown
}
for (int i = 0; i < workers.Count; i++)
{
workers.ElementAt(i).currentTime = currentTime;
}
if (rows.Count > 0)
{
for (int i = 0; i < rows.Count; i++)
{
counter++;
InputDTO dto = new InputDTO((String)rows.ElementAt(i), counter);
putToWorkerQ(dto);
if (i > 0 && i % workers.Count == 0)
{
Thread.Sleep(queueLoadingPauseTimeValue);
}
}
}
currentTime = CommonConstants.currentTimeMillis();
saveTimeDelta = 0;
if (!workersStarted)
{
// set startTime again for closer approx of actual time to process batch
startTime = CommonConstants.currentTimeMillis();
for (int i = 0; i < workers.Count; i++)
{
workers.ElementAt(i).start();
}
workersStarted = true;
}
else
{
for (int i = 0; i < workers.Count; i++)
{
// start any new ones
if (!workers.ElementAt(i).isRunning())
{
workers.ElementAt(i).start();
}
else
{
// reset currentTime for workers
// they can then sleep for the balance of the second if queue goes empty
workers.ElementAt(i).currentTime = currentTime;
}
}
}
// allow the workers to do some work
Thread.Sleep(waitForWorkersTime); // this should not be made too small
timeDelta = (int)(CommonConstants.currentTimeMillis() - currentTime);
//************************************//
while (timeDelta < almost1Second)
{
// this is our housekeeping loop
// process the output from the workers, update the database
if (allWorkersIdle())
{
if (saveTimeDelta == 0)
{
saveTimeDelta = timeDelta; // used to calculate burst rate
}
if (!stopping && !tooManyWorkers && timeDelta < processingTooFastTime)
{
tooManyWorkers = true;
Console.WriteLine(CommonConstants.currentTimeExtended() + ": too many workers @ " + timeDelta);
}
}
if (getResultQLength() > 0)
{
// process 1 entry in our input q
processResultQItem();
}
else
{
Thread.Sleep(housekeepingSleepTimeValue);
}
timeDelta = (int)(CommonConstants.currentTimeMillis() - currentTime);
}
//************************************//
// almost 1 second has elapsed.
if (stopping) // not a panic abort, but orderly shutdown
{
while (!allWorkersIdle() || getResultQLength() > 0)
{
if (getResultQLength() > 0)
{
// process 1 entry in our input q
processResultQItem();
}
else
{
Thread.Sleep(housekeepingSleepTimeValue);
}
}
// if we get here, all Qs are empty
// we can now shut down the workers
updateTimestamp = CommonConstants.currentTimeMillis();
long shutdownAbort = updateTimestamp + maxShutdownTimeValue;
for (int i = 0; i < workers.Count; i++)
{
workers.ElementAt(i).stop();
}
Thread.Sleep(300);
bool threadsStillRunning = true;
while (threadsStillRunning)
{
if (CommonConstants.currentTimeMillis() > shutdownAbort)
{
for (int i = 0; i < workers.Count; i++)
{
if (workers.ElementAt(i).isRunning())
{
workers.ElementAt(i).stopNow();
}
}
}
else
{
threadsStillRunning = false;
for (int i = 0; i < workers.Count; i++)
{
if (workers.ElementAt(i).isRunning())
{
threadsStillRunning = true;
}
}
}
if (threadsStillRunning)
{
Thread.Sleep(100);
}
}
}
else
{
// not stopping, almost 1 second elapsed.
// do we need more or less workers?
wqLen = getWorkerQLength();
burst -= wqLen;
if (saveTimeDelta != 0)
{
Console.WriteLine(CommonConstants.currentTimeExtended() + ": processed " + burst + " items in " + saveTimeDelta + " ms, loop time=" + timeDelta);
}
else
{
Console.WriteLine(CommonConstants.currentTimeExtended() + ": processed " + burst + " items in " + timeDelta + " ms");
saveTimeDelta = timeDelta;
}
if (wqLen > 0 || saveTimeDelta > processingTooSlowTime)
{
needMoreWorkers = true;
Console.WriteLine(CommonConstants.currentTimeExtended() + ": more workers needed Q = " + wqLen + ", tDelta=" + saveTimeDelta);
}
if (needMoreWorkers)
{
int nbrNeeded = wqLen / workers.Count;
if (nbrNeeded == 0)
{
nbrNeeded = 1;
}
for (int i = 0; i < nbrNeeded; i++)
{
if (workers.Count < maxThreads)
{
// do not start them until we give them work
int n = workers.Count;
MultiThreadWorker mtw = new MultiThreadWorker("Worker " + n, workerQueue, resultQueue);
workers.Add(mtw);
}
}
needMoreWorkers = false;
tooManyWorkers = false;
}
if (tooManyWorkers)
{
if (workers.Count > minThreads && saveTimeDelta > 0 && wqLen == 0)
{
// remove 1 worker
int n = workers.Count - 1;
MultiThreadWorker mtw = workers.ElementAt(n);
workers.RemoveAt(n);
mtw.stop(); // this allows it to shut down in an orderly fashion
// and return its result if it is processing
}
tooManyWorkers = false;
}
}
} catch (ThreadInterruptedException ex) {
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread was interrupted : " + ex.Message);
}
}
closeDBConnection(StopFileExists(serviceStopFile));
if (StopFileExists(serviceStopFile))
{
System.IO.File.Delete(serviceStopFile);
}
long deltaT = updateTimestamp - startTime;
double decSecs = ((double)deltaT) / 1000;
int seconds = (int)(deltaT / 1000);
double average = Math.Floor(((double)counter * 1000) / ((double)deltaT) * 100) / 100; // per second 2 decimal places
Console.WriteLine(CommonConstants.currentTimeExtended() + ": #### - Stopping - processed " + counter + " requests in " + decSecs + " seconds" +
" ==> average = " + average + " per second");
}
///////////////////////////////////////////////
/// work is done here
///
public void doWork()
{
Random rand = new Random();
InputDTO dto = null;
ServiceResult res = null;
OutputDTO outDTO = null;
running = true;
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Running " + threadName);
try {
currentTime = CommonConstants.currentTimeMillis();
while (!stopping)
{
int pauseTime = rand.Next(71) + 170; // random web request time 170..240
if (!workerQueue.IsEmpty)
{
// prevent blocking loop by checking Q length first
dto = getFromWorkerQ(); // may return null
if (dto != null)
{
// simulate processing = Let the thread sleep for a while.
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread: " + threadName + ", row " + dto.RowNbr + ", Qlen = " + getQLength() +
", working for " + pauseTime);
Thread.Sleep(pauseTime);
// send back the results
outDTO = new OutputDTO(dto);
res = new ServiceResult(CommonConstants.statusSuccess, outDTO);
putToResultQ(res);
}
}
if (getQLength() == 0)
{
long endTime = CommonConstants.currentTimeMillis();
int deltaT = (int)(endTime - currentTime);
if (deltaT < 100) // less than a tenth second has elapsed
{
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread: " + threadName + ", Qlen = " + getQLength() + ", sleeping for " + sleepAdj);
Thread.Sleep(sleepAdj);
}
else if (deltaT < almost1Second) // less than a second has elapsed
{
sleepTime = 1000 - deltaT - sleepAdj;
if (sleepTime < 0)
{
sleepTime = smallSleepTime;
}
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread: " + threadName + ", Qlen = " + getQLength() + ", sleeping for " + sleepTime);
Thread.Sleep(sleepTime);
}
else
{
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread: " + threadName + ", Qlen = " + getQLength() + ", elapsed " + deltaT + ", sleeping for " + smallSleepTime);
Thread.Sleep(smallSleepTime);
}
//currentTime = System.currentTimeMillis();
}
}
} catch (ThreadInterruptedException e) {
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread " + threadName + " interrupted.");
}
Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread " + threadName + " exiting.");
running = false;
}