예제 #1
0
        /// <summary>
        /// We will set this when we need more updates, but none are present in the input queue.
        /// </summary>
        //public AutoResetEvent moreUpdatesRequiredEvent;

        public downloadThread(Program.logString logger, AutoResetEvent pollEvent, ConcurrentQueue <downloadedUpdate> outqueue, updateReserverThread updateReserver)
            : base(logger, pollEvent)
        {
            threadname          = "Download thread";
            this.outqueue       = outqueue;
            this.updateReserver = updateReserver;
        }
예제 #2
0
        public static void Main(string[] args)
        {
            using (logDB = new updateDB(connstr, dbName))
            {
                ConcurrentQueue <downloadThread.downloadedUpdate> toProcessQueue =
                    new ConcurrentQueue <downloadThread.downloadedUpdate>();
                ConcurrentQueue <string> toArchiveQueue        = new ConcurrentQueue <string>();
                AutoResetEvent           downloadRequestEvent  = new AutoResetEvent(false);
                AutoResetEvent           downloadCompleteEvent = new AutoResetEvent(false);

                using (threadLifetimeCollection threads = new threadLifetimeCollection())
                {
                    asyncsqlthread sqlparams = new asyncsqlthread(logger);
                    threads.add(sqlparams.start());

                    updateReserverThread updateGetterThread = new updateReserverThread(logger);
                    threads.add(updateGetterThread.start());

                    archiveThread archiver = new archiveThread(logger, toArchiveQueue);
                    threads.add(archiver.start());

                    downloadThread[] downloadThreads = new downloadThread[downloadThreadCount];
                    for (int i = 0; i < downloadThreads.Length; i++)
                    {
                        downloadThreads[i] =
                            new downloadThread(logger, downloadRequestEvent, toProcessQueue, updateGetterThread);
                        downloadThreads[i].threadname += $" ({i})";

                        downloadThreads[i].outqueue             = toProcessQueue;
                        downloadThreads[i].maxFinishedDownloads = maxPendingDownloads;
                        downloadThreads[i].downloadComplete     = downloadCompleteEvent;

                        threads.add(downloadThreads[i].start());
                    }

                    processingThread[] processingThreads = new processingThread[processingThreadCount];
                    for (int i = 0; i < processingThreads.Length; i++)
                    {
                        processingThreads[i] = new processingThread(logger, downloadCompleteEvent, toProcessQueue,
                                                                    downloadRequestEvent, archiver, sqlparams);
                        processingThreads[i].threadname += $" ({i})";

                        threads.add(processingThreads[i].start());
                    }

                    while (true)
                    {
                        // have we finished it all?
                        if (updateGetterThread.allUpdatesExhausted())
                        {
                            Console.WriteLine("all done");
                            // Once the update-getter signals completion, it has already passed pending updates to the download threads, so it's safe to
                            // tear down threads without fear of missing updates.
                            foreach (downloadThread dt in downloadThreads)
                            {
                                dt.stop();
                            }
                            foreach (processingThread pt in processingThreads)
                            {
                                pt.stop();
                            }
                            archiver.stop();
                            sqlparams.stop();

                            break;
                        }

                        // OK, show some stats.
                        Thread.Sleep(TimeSpan.FromSeconds(1));
                        //Console.Clear();
                        Console.WriteLine(
                            $"Processing queue: {toProcessQueue.Count}, reserved updates {updateGetterThread.claimedUpdates.Count}, " +
                            $"async sql queue {sqlparams.batches.Count} / {sqlparams.batches_wim.Count} / {sqlparams.toMarkComplete.Count}");

                        for (int i = 0; i < downloadThreads.Length; i++)
                        {
                            var current = downloadThreads[i].currentlydownloading;
                            if (current != null)
                            {
                                Console.WriteLine(
                                    $"Download thread {i}: downloading {current.downloadURI} ({current.sizeMB} MB)");
                            }
                            else
                            {
                                Console.WriteLine($"Download thread {i}: idle");
                            }
                        }

                        for (int i = 0; i < processingThreads.Length; i++)
                        {
                            string cur = processingThreads[i].currentlyProcessing;
                            if (cur != null)
                            {
                                Console.WriteLine($"Processing thread {i}: processing {cur}");
                            }
                            else
                            {
                                Console.WriteLine($"Processing thread {i}: idle");
                            }
                        }
                    }
                }
            }
        }