public processingThread(Program.logString logger, AutoResetEvent poller,
ConcurrentQueue <downloadThread.downloadedUpdate> inputQueue, AutoResetEvent downloadRequestEvent,
archiveThread archiver,
asyncsqlthread sqlThread) : base(logger, poller)
{
this.threadname = "Update processing thread";
this.queue = inputQueue;
this.asyncSQLParams = sqlThread;
this.archiver = archiver;
this.UpdateProcessingStarted = downloadRequestEvent;
}
public static void Main(string[] args)
{
using (logDB = new updateDB(connstr, dbName))
{
ConcurrentQueue <downloadThread.downloadedUpdate> toProcessQueue =
new ConcurrentQueue <downloadThread.downloadedUpdate>();
ConcurrentQueue <string> toArchiveQueue = new ConcurrentQueue <string>();
AutoResetEvent downloadRequestEvent = new AutoResetEvent(false);
AutoResetEvent downloadCompleteEvent = new AutoResetEvent(false);
using (threadLifetimeCollection threads = new threadLifetimeCollection())
{
asyncsqlthread sqlparams = new asyncsqlthread(logger);
threads.add(sqlparams.start());
updateReserverThread updateGetterThread = new updateReserverThread(logger);
threads.add(updateGetterThread.start());
archiveThread archiver = new archiveThread(logger, toArchiveQueue);
threads.add(archiver.start());
downloadThread[] downloadThreads = new downloadThread[downloadThreadCount];
for (int i = 0; i < downloadThreads.Length; i++)
{
downloadThreads[i] =
new downloadThread(logger, downloadRequestEvent, toProcessQueue, updateGetterThread);
downloadThreads[i].threadname += $" ({i})";
downloadThreads[i].outqueue = toProcessQueue;
downloadThreads[i].maxFinishedDownloads = maxPendingDownloads;
downloadThreads[i].downloadComplete = downloadCompleteEvent;
threads.add(downloadThreads[i].start());
}
processingThread[] processingThreads = new processingThread[processingThreadCount];
for (int i = 0; i < processingThreads.Length; i++)
{
processingThreads[i] = new processingThread(logger, downloadCompleteEvent, toProcessQueue,
downloadRequestEvent, archiver, sqlparams);
processingThreads[i].threadname += $" ({i})";
threads.add(processingThreads[i].start());
}
while (true)
{
// have we finished it all?
if (updateGetterThread.allUpdatesExhausted())
{
Console.WriteLine("all done");
// Once the update-getter signals completion, it has already passed pending updates to the download threads, so it's safe to
// tear down threads without fear of missing updates.
foreach (downloadThread dt in downloadThreads)
{
dt.stop();
}
foreach (processingThread pt in processingThreads)
{
pt.stop();
}
archiver.stop();
sqlparams.stop();
break;
}
// OK, show some stats.
Thread.Sleep(TimeSpan.FromSeconds(1));
//Console.Clear();
Console.WriteLine(
$"Processing queue: {toProcessQueue.Count}, reserved updates {updateGetterThread.claimedUpdates.Count}, " +
$"async sql queue {sqlparams.batches.Count} / {sqlparams.batches_wim.Count} / {sqlparams.toMarkComplete.Count}");
for (int i = 0; i < downloadThreads.Length; i++)
{
var current = downloadThreads[i].currentlydownloading;
if (current != null)
{
Console.WriteLine(
$"Download thread {i}: downloading {current.downloadURI} ({current.sizeMB} MB)");
}
else
{
Console.WriteLine($"Download thread {i}: idle");
}
}
for (int i = 0; i < processingThreads.Length; i++)
{
string cur = processingThreads[i].currentlyProcessing;
if (cur != null)
{
Console.WriteLine($"Processing thread {i}: processing {cur}");
}
else
{
Console.WriteLine($"Processing thread {i}: idle");
}
}
}
}
}
}