public static void Main(string[] args)
{
using (logDB = new updateDB(connstr, dbName))
{
ConcurrentQueue <downloadThread.downloadedUpdate> toProcessQueue =
new ConcurrentQueue <downloadThread.downloadedUpdate>();
ConcurrentQueue <string> toArchiveQueue = new ConcurrentQueue <string>();
AutoResetEvent downloadRequestEvent = new AutoResetEvent(false);
AutoResetEvent downloadCompleteEvent = new AutoResetEvent(false);
using (threadLifetimeCollection threads = new threadLifetimeCollection())
{
asyncsqlthread sqlparams = new asyncsqlthread(logger);
threads.add(sqlparams.start());
updateReserverThread updateGetterThread = new updateReserverThread(logger);
threads.add(updateGetterThread.start());
archiveThread archiver = new archiveThread(logger, toArchiveQueue);
threads.add(archiver.start());
downloadThread[] downloadThreads = new downloadThread[downloadThreadCount];
for (int i = 0; i < downloadThreads.Length; i++)
{
downloadThreads[i] =
new downloadThread(logger, downloadRequestEvent, toProcessQueue, updateGetterThread);
downloadThreads[i].threadname += $" ({i})";
downloadThreads[i].outqueue = toProcessQueue;
downloadThreads[i].maxFinishedDownloads = maxPendingDownloads;
downloadThreads[i].downloadComplete = downloadCompleteEvent;
threads.add(downloadThreads[i].start());
}
processingThread[] processingThreads = new processingThread[processingThreadCount];
for (int i = 0; i < processingThreads.Length; i++)
{
processingThreads[i] = new processingThread(logger, downloadCompleteEvent, toProcessQueue,
downloadRequestEvent, archiver, sqlparams);
processingThreads[i].threadname += $" ({i})";
threads.add(processingThreads[i].start());
}
while (true)
{
// have we finished it all?
if (updateGetterThread.allUpdatesExhausted())
{
Console.WriteLine("all done");
// Once the update-getter signals completion, it has already passed pending updates to the download threads, so it's safe to
// tear down threads without fear of missing updates.
foreach (downloadThread dt in downloadThreads)
{
dt.stop();
}
foreach (processingThread pt in processingThreads)
{
pt.stop();
}
archiver.stop();
sqlparams.stop();
break;
}
// OK, show some stats.
Thread.Sleep(TimeSpan.FromSeconds(1));
//Console.Clear();
Console.WriteLine(
$"Processing queue: {toProcessQueue.Count}, reserved updates {updateGetterThread.claimedUpdates.Count}, " +
$"async sql queue {sqlparams.batches.Count} / {sqlparams.batches_wim.Count} / {sqlparams.toMarkComplete.Count}");
for (int i = 0; i < downloadThreads.Length; i++)
{
var current = downloadThreads[i].currentlydownloading;
if (current != null)
{
Console.WriteLine(
$"Download thread {i}: downloading {current.downloadURI} ({current.sizeMB} MB)");
}
else
{
Console.WriteLine($"Download thread {i}: idle");
}
}
for (int i = 0; i < processingThreads.Length; i++)
{
string cur = processingThreads[i].currentlyProcessing;
if (cur != null)
{
Console.WriteLine($"Processing thread {i}: processing {cur}");
}
else
{
Console.WriteLine($"Processing thread {i}: idle");
}
}
}
}
}
}
protected override void threadmain()
{
WebRequest.DefaultWebProxy = null;
while (true)
{
if (updateReserver.allUpdatesExhausted())
{
if (exitTime)
{
break;
}
pollTimer.WaitOne(100);
continue;
}
// Keep a maximum of this many files on disk
if (outqueue.Count > maxFinishedDownloads)
{
pollTimer.WaitOne(100);
continue;
}
// Get an update to download from the input queue
if (updateReserver.tryDequeueUpdate(out wsusUpdate toDownload) == false)
{
continue;
}
currentlydownloading = toDownload;
string outputfile = null;
// If the file already exists in the archive or temp dir, use it from that location.
string tempfile = Path.Combine(Program.tempdir, toDownload.getTemporaryFilename());
if (File.Exists(tempfile))
{
outputfile = tempfile;
}
if (Program.archivedir != null)
{
string archivefile = Path.Combine(Program.archivedir, toDownload.getTemporaryFilename());
if (File.Exists(archivefile))
{
outputfile = archivefile;
}
}
if (File.Exists(outputfile))
{
// The file already exists. Let's check its length and hash, just to be sure it's what we want.
if (verifyUpdateFile(outputfile, toDownload) == false)
{
// It's corrupt, or partially downloaded. Delete it and we'll start again.
archiveThread.deleteWithRetry(outputfile, OnLogString);
outputfile = null;
}
}
// Now we should download the file if necessary, into the temporary file location.
while (outputfile == null)
{
try
{
using (WebClient client = new WebClient())
{
client.Proxy = null;
client.DownloadFile(toDownload.downloadURI, tempfile);
currentlydownloading = null;
outputfile = tempfile;
}
}
catch (WebException e)
{
logString(e, $"Download of {toDownload.downloadURI} to {outputfile} failed");
Thread.Sleep(TimeSpan.FromSeconds(10));
}
}
// Finally, pass the downloaded update off to the processing thread queue.
try
{
downloadedUpdate newlyDownloaded = new downloadedUpdate();
newlyDownloaded.absolutepath = outputfile;
newlyDownloaded.update = wsusUpdate.parse(toDownload, outputfile);
outqueue.Enqueue(newlyDownloaded);
downloadComplete.Set();
}
catch (Exception e)
{
logString(e, $"Processing of update from {toDownload.downloadURI} as {outputfile} failed");
}
}
}