static void Main(string[] args)
        {
            MultiThreadOrchestrator orchestrator = new MultiThreadOrchestrator();

            orchestrator.Orchestrate();
            Console.WriteLine(CommonConstants.currentTimeExtended() + ": Processing Finished");
        }
 public MultiThreadWorker(String name, ConcurrentQueue <InputDTO> wQ, ConcurrentQueue <ServiceResult> rQ)
 {
     threadName = name;
     Console.WriteLine(CommonConstants.currentTimeExtended() + ": Creating " + threadName);
     workerQueue = wQ;
     resultQueue = rQ;
 }
 private InputDTO getFromWorkerQ()
 {
     lock (workerQueue) {
         if (workerQueue.Count() > 0)
         {
             // take() and poll(timeout) can block
             // the timeout is erratic - often at least 5 to 10 times the value
             // polling can cause a blocking loop that can run for up to 500ms for all workers
             // by checking the Q length before trying take(), we generally
             // have sufficient rows to satisfy all workers.
             long     t1 = CommonConstants.currentTimeMicros();
             InputDTO dto;
             bool     gotit = workerQueue.TryDequeue(out dto);
             long     t2    = CommonConstants.currentTimeMicros() - t1;
             if (t2 > 10)
             { // 10th of milisecond
                 Console.WriteLine(CommonConstants.currentTimeExtended() + ": " + threadName + " dequeue timeout=" + t2 + " micros");
             }
             if (gotit)
             {
                 isProcessing = true;
                 return(dto);
             }
             else
             {
                 return(null);
             }
         }
         else
         {
             return(null);
         }
     }
 }
 public void start()
 {
     Console.WriteLine(CommonConstants.currentTimeExtended() + ": Starting " + threadName);
     if (t == null)
     {
         t = new Thread(doWork);
         t.Start();
     }
 }
        private void processResultQItem()
        {
            // process 1 entry in our input q
            Console.WriteLine(CommonConstants.currentTimeExtended() + ": processing ResultQ len=" + getResultQLength());
            ServiceResult sr = getFromResultQ();

            if (sr != null)  // do stuff with it
            {
                //Console.WriteLine(CommonConstants.currentTimeExtended() + ": res " + sr.getResult() + ", rec=" + sr.getDto().RowNbr + ", data [" + sr.getDto().Data + "]");
            }
            Thread.Sleep(pseudoProcessingSleepTimeValue);
        }
        private void performInit()
        {
            Console.WriteLine(CommonConstants.currentTimeExtended() + ": ##### Starting Application");

            //readConfigFile(configFilePath);

            // Update 'urlSuffix' value to include 'clientID'
            //urlSuffix = urlSuffix + "&client=" + clientID;

            // Create Service 'StopFile'
            //File stopFile = new File(serviceStopFile);

            //return stopFile;
        }
 private void closeDBConnection(bool stopFile)
 {
     if (stopFile)
     {
         Console.WriteLine(CommonConstants.currentTimeExtended() + ": Finishing Application - 'Stop' File was found...");
     }
     else
     {
         Console.WriteLine(CommonConstants.currentTimeExtended() + ": Finishing Application - Something called 'exit()'...");
     }
     if (simulateDB)
     {
         try
         {
             dbOutput.Close();
         }
         catch (IOException e)
         {
             // TODO Auto-generated catch block
             Console.WriteLine(e.StackTrace);
         }
     }
 }
        public void Orchestrate()
        {
            performInit();
            openDBConnection();
            int counter = 0;
            int burst   = 0;
            int wqLen   = 0;

            // ////////////////////////////////////////////
            workerQueue = new ConcurrentQueue <InputDTO>();
            resultQueue = new ConcurrentQueue <ServiceResult>();
            workers     = new List <MultiThreadWorker>(maxThreads);
            for (int i = 0; i < nbrThreadsInit; i++)
            {
                workers.Add(new MultiThreadWorker("Worker " + i, workerQueue, resultQueue));
            }

            // ////////////////////////////////////////////

            long currentTime   = CommonConstants.currentTimeMillis();
            long startTime     = currentTime;
            int  timeDelta     = 0;
            int  saveTimeDelta = 0;

            while (!stopping)
            {
                // we should come through here once per second
                // almost1Second is exactly that - if we construct the loop for exactly 1 second,
                // we go past that value by as much as 35 ms
                // recalculate it here, because we might adjust the housekeepingSleepTimeValue
                almost1Second = 1000 - housekeepingSleepTimeValue + 1;

                // read dbOutput requestsPerSecond rows at a time, and apportion to workers
                // if the workerQueue contains rows, read less rows to ensure the queue total
                // will be the configured batch size
                try {
                    List <String> rows = performQuery(wqLen);                 // could set stopping = true

                    Console.WriteLine(CommonConstants.currentTimeExtended() + ": read " + rows.Count + " qLen = " + wqLen);
                    burst = rows.Count + getWorkerQLength();                 // how many we are trying to process this second
                    if (StopFileExists(serviceStopFile))
                    {
                        stopping = true;                       // only set flag to allow orderly shutdown
                    }
                    for (int i = 0; i < workers.Count; i++)
                    {
                        workers.ElementAt(i).currentTime = currentTime;
                    }
                    if (rows.Count > 0)
                    {
                        for (int i = 0; i < rows.Count; i++)
                        {
                            counter++;
                            InputDTO dto = new InputDTO((String)rows.ElementAt(i), counter);
                            putToWorkerQ(dto);
                            if (i > 0 && i % workers.Count == 0)
                            {
                                Thread.Sleep(queueLoadingPauseTimeValue);
                            }
                        }
                    }
                    currentTime   = CommonConstants.currentTimeMillis();
                    saveTimeDelta = 0;
                    if (!workersStarted)
                    {
                        // set startTime again for closer approx of actual time to process batch
                        startTime = CommonConstants.currentTimeMillis();
                        for (int i = 0; i < workers.Count; i++)
                        {
                            workers.ElementAt(i).start();
                        }
                        workersStarted = true;
                    }
                    else
                    {
                        for (int i = 0; i < workers.Count; i++)
                        {
                            // start any new ones
                            if (!workers.ElementAt(i).isRunning())
                            {
                                workers.ElementAt(i).start();
                            }
                            else
                            {
                                // reset currentTime for workers
                                // they can then sleep for the balance of the second if queue goes empty
                                workers.ElementAt(i).currentTime = currentTime;
                            }
                        }
                    }

                    // allow the workers to do some work
                    Thread.Sleep(waitForWorkersTime);                 // this should not be made too small

                    timeDelta = (int)(CommonConstants.currentTimeMillis() - currentTime);

                    //************************************//
                    while (timeDelta < almost1Second)
                    {
                        // this is our housekeeping loop
                        // process the output from the workers, update the database
                        if (allWorkersIdle())
                        {
                            if (saveTimeDelta == 0)
                            {
                                saveTimeDelta = timeDelta;     // used to calculate burst rate
                            }
                            if (!stopping && !tooManyWorkers && timeDelta < processingTooFastTime)
                            {
                                tooManyWorkers = true;
                                Console.WriteLine(CommonConstants.currentTimeExtended() + ": too many workers @ " + timeDelta);
                            }
                        }
                        if (getResultQLength() > 0)
                        {
                            // process 1 entry in our input q
                            processResultQItem();
                        }
                        else
                        {
                            Thread.Sleep(housekeepingSleepTimeValue);
                        }
                        timeDelta = (int)(CommonConstants.currentTimeMillis() - currentTime);
                    }
                    //************************************//

                    // almost 1 second has elapsed.

                    if (stopping)                    // not a panic abort, but orderly shutdown
                    {
                        while (!allWorkersIdle() || getResultQLength() > 0)
                        {
                            if (getResultQLength() > 0)
                            {
                                // process 1 entry in our input q
                                processResultQItem();
                            }
                            else
                            {
                                Thread.Sleep(housekeepingSleepTimeValue);
                            }
                        }
                        // if we get here, all Qs are empty
                        // we can now shut down the workers
                        updateTimestamp = CommonConstants.currentTimeMillis();
                        long shutdownAbort = updateTimestamp + maxShutdownTimeValue;
                        for (int i = 0; i < workers.Count; i++)
                        {
                            workers.ElementAt(i).stop();
                        }
                        Thread.Sleep(300);
                        bool threadsStillRunning = true;
                        while (threadsStillRunning)
                        {
                            if (CommonConstants.currentTimeMillis() > shutdownAbort)
                            {
                                for (int i = 0; i < workers.Count; i++)
                                {
                                    if (workers.ElementAt(i).isRunning())
                                    {
                                        workers.ElementAt(i).stopNow();
                                    }
                                }
                            }
                            else
                            {
                                threadsStillRunning = false;
                                for (int i = 0; i < workers.Count; i++)
                                {
                                    if (workers.ElementAt(i).isRunning())
                                    {
                                        threadsStillRunning = true;
                                    }
                                }
                            }
                            if (threadsStillRunning)
                            {
                                Thread.Sleep(100);
                            }
                        }
                    }
                    else
                    {
                        // not stopping, almost 1 second elapsed.
                        // do we need more or less workers?
                        wqLen  = getWorkerQLength();
                        burst -= wqLen;
                        if (saveTimeDelta != 0)
                        {
                            Console.WriteLine(CommonConstants.currentTimeExtended() + ": processed " + burst + " items in " + saveTimeDelta + " ms, loop time=" + timeDelta);
                        }
                        else
                        {
                            Console.WriteLine(CommonConstants.currentTimeExtended() + ": processed " + burst + " items in " + timeDelta + " ms");
                            saveTimeDelta = timeDelta;
                        }
                        if (wqLen > 0 || saveTimeDelta > processingTooSlowTime)
                        {
                            needMoreWorkers = true;
                            Console.WriteLine(CommonConstants.currentTimeExtended() + ": more workers needed Q = " + wqLen + ", tDelta=" + saveTimeDelta);
                        }
                        if (needMoreWorkers)
                        {
                            int nbrNeeded = wqLen / workers.Count;
                            if (nbrNeeded == 0)
                            {
                                nbrNeeded = 1;
                            }
                            for (int i = 0; i < nbrNeeded; i++)
                            {
                                if (workers.Count < maxThreads)
                                {
                                    // do not start them until we give them work
                                    int n = workers.Count;
                                    MultiThreadWorker mtw = new MultiThreadWorker("Worker " + n, workerQueue, resultQueue);
                                    workers.Add(mtw);
                                }
                            }
                            needMoreWorkers = false;
                            tooManyWorkers  = false;
                        }
                        if (tooManyWorkers)
                        {
                            if (workers.Count > minThreads && saveTimeDelta > 0 && wqLen == 0)
                            {
                                // remove 1 worker
                                int n = workers.Count - 1;
                                MultiThreadWorker mtw = workers.ElementAt(n);
                                workers.RemoveAt(n);
                                mtw.stop();                 // this allows it to shut down in an orderly fashion
                                                            // and return its result if it is processing
                            }
                            tooManyWorkers = false;
                        }
                    }
                } catch (ThreadInterruptedException ex) {
                    Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread was interrupted : " + ex.Message);
                }
            }
            closeDBConnection(StopFileExists(serviceStopFile));
            if (StopFileExists(serviceStopFile))
            {
                System.IO.File.Delete(serviceStopFile);
            }

            long   deltaT  = updateTimestamp - startTime;
            double decSecs = ((double)deltaT) / 1000;
            int    seconds = (int)(deltaT / 1000);
            double average = Math.Floor(((double)counter * 1000) / ((double)deltaT) * 100) / 100;               // per second 2 decimal places

            Console.WriteLine(CommonConstants.currentTimeExtended() + ":  #### - Stopping - processed " + counter + " requests in " + decSecs + " seconds" +
                              " ==> average = " + average + " per second");
        }
        ///////////////////////////////////////////////
        /// work is done here
        ///

        public void doWork()
        {
            Random        rand   = new Random();
            InputDTO      dto    = null;
            ServiceResult res    = null;
            OutputDTO     outDTO = null;

            running = true;
            Console.WriteLine(CommonConstants.currentTimeExtended() + ": Running " + threadName);
            try {
                currentTime = CommonConstants.currentTimeMillis();
                while (!stopping)
                {
                    int pauseTime = rand.Next(71) + 170; // random web request time 170..240
                    if (!workerQueue.IsEmpty)
                    {
                        // prevent blocking loop by checking Q length first
                        dto = getFromWorkerQ();   // may return null
                        if (dto != null)
                        {
                            // simulate processing = Let the thread sleep for a while.
                            Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread: " + threadName + ", row " + dto.RowNbr + ", Qlen = " + getQLength() +
                                              ", working for " + pauseTime);
                            Thread.Sleep(pauseTime);
                            // send back the results
                            outDTO = new OutputDTO(dto);
                            res    = new ServiceResult(CommonConstants.statusSuccess, outDTO);
                            putToResultQ(res);
                        }
                    }
                    if (getQLength() == 0)
                    {
                        long endTime = CommonConstants.currentTimeMillis();
                        int  deltaT  = (int)(endTime - currentTime);
                        if (deltaT < 100)   // less than a tenth second has elapsed
                        {
                            Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread: " + threadName + ", Qlen = " + getQLength() + ", sleeping for " + sleepAdj);
                            Thread.Sleep(sleepAdj);
                        }
                        else if (deltaT < almost1Second)     // less than a second has elapsed
                        {
                            sleepTime = 1000 - deltaT - sleepAdj;
                            if (sleepTime < 0)
                            {
                                sleepTime = smallSleepTime;
                            }
                            Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread: " + threadName + ", Qlen = " + getQLength() + ", sleeping for " + sleepTime);
                            Thread.Sleep(sleepTime);
                        }
                        else
                        {
                            Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread: " + threadName + ", Qlen = " + getQLength() + ", elapsed " + deltaT + ", sleeping for " + smallSleepTime);
                            Thread.Sleep(smallSleepTime);
                        }
                        //currentTime = System.currentTimeMillis();
                    }
                }
            } catch (ThreadInterruptedException e) {
                Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread " + threadName + " interrupted.");
            }
            Console.WriteLine(CommonConstants.currentTimeExtended() + ": Thread " + threadName + " exiting.");
            running = false;
        }