public bulkDBInsert(wsusUpdate parent, asyncsqlthread db)
{
_parent = parent;
_db = db;
filesInBatchSoFar = 0;
files = new T[80];
}
public file(wsusUpdate parent, updateFile src)
{
this.sourceFilename = parent.filename;
this.filename = src.filename;
this.filelocation = src.filelocation;
this.fileextension = src.fileextension;
this.hash_sha256 = src.hash_sha256;
this.size = (long)src.size;
this.pe_sizeOfCode = toNullableBinary(src.pe_sizeOfCode);
this.pe_timestamp = toNullableBinary(src.pe_timedatestamp);
this.pe_magicType = toNullableBinary(src.pe_magicType);
this.contents128b = src.contents128b;
this.rsds_GUID = src.rsds_GUID;
this.rsds_age = (int)src.rsds_age;
this.rsds_filename = src.rsds_filename;
this.authenticode_certfriendly = src.authenticode_certfriendly;
this.authenticode_certsubj = src.authenticode_certsubj;
this.FileDescription = src.FileDescription;
this.FileVersion = src.FileVersion;
this.ProductName = src.ProductName;
this.ProductVersion = src.ProductVersion;
this.Comments = src.Comments;
this.CompanyName = src.CompanyName;
}
public wsusUpdate(wsusUpdate src)
{
this.fileHashFromWSUS = src.fileHashFromWSUS;
this.downloadURI = src.downloadURI;
this.sizeBytes = src.sizeBytes;
this.filename = getTemporaryFilename();
}
public void bulkInsertFiles(wsusUpdate grandparent, IEnumerable <file> toInsertList)
{
using (SqlCommand cmd = _dbConn.CreateCommand())
{
string initialsql = $"Declare @testdata as FileTableType; ";
cmd.CommandText = initialsql;
int thisbatchcnt = 0;
void flushBatch()
{
addParamWithValue(cmd, $"@parentFileHash", grandparent.fileHashFromWSUS);
cmd.CommandText += " exec insertfiles @parentFileHash,@testdata; ";
cmd.ExecuteNonQuery();
cmd.CommandText = $"Declare @testdata as FileTableType; ";
thisbatchcnt = 0;
cmd.Parameters.Clear();
}
foreach (file toInsert in toInsertList)
{
addFileToBatchCommand(toInsert, thisbatchcnt, cmd);
thisbatchcnt++;
if (thisbatchcnt == 80)
{
flushBatch();
}
}
flushBatch();
}
}
public fileSource_cab(wsusUpdate parent, wsusUpdate_cab src)
{
this.wsusFileID = parent.dbID.Value;
this.offlineCapable = src.offlineCapable;
this.description = src.description;
this.productName = src.productName;
this.supportInfo = src.supportInfo;
}
public void insertOrCreateWsusFileNoUpdate(wsusUpdate toInsert)
{
if (toInsert.dbID.HasValue)
{
return;
}
insertOrCreateNoUpdate(toInsert, "wsusFile", new [] { "filename" });
}
public void insertOrCreateCabFileNoUpdate(wsusUpdate parent, wsusUpdate_cab metadata)
{
if (parent.dbID == null)
{
insertOrCreateWsusFileNoUpdate(parent);
}
insert_noconcurrency(new [] { new fileSource_cab(parent, metadata) }, "filesource_cab");
}
public bool tryDequeueUpdate(out wsusUpdate outputOrNull)
{
bool toRet = claimedUpdates.TryDequeue(out outputOrNull);
if (toRet == false || outputOrNull == null)
{
// If we are stalling, ask the mail thread for more update IDs.
pollTimer.Set();
}
return(toRet);
}
protected override void threadmain()
{
using (updateDB db = new updateDB(Program.connstr, Program.dbName))
{
// Read anything we queued in a previous run
foreach (var alreadyClaimed in db.getClaimedUpdate())
{
claimedUpdates.Enqueue(alreadyClaimed);
}
// Make sure we have at least highWaterMark updates queued (if they are available)
while (claimedUpdates.Count < highWaterMark)
{
wsusUpdate nextUpdate = db.startNextUpdate();
if (nextUpdate == null)
{
_allUpdatesExhausted = true;
exitTime = true;
break;
}
claimedUpdates.Enqueue(nextUpdate);
}
while (true)
{
// if we drop below the low water mark, queue updates until we get to the high water mark.
if (claimedUpdates.Count < lowWaterMark)
{
while (claimedUpdates.Count < highWaterMark)
{
wsusUpdate nextUpdate = db.startNextUpdate();
if (nextUpdate == null)
{
_allUpdatesExhausted = true;
exitTime = true;
break;
}
claimedUpdates.Enqueue(nextUpdate);
}
}
if (exitTime)
{
// TODO: should we unmark what we have in progress before exiting?
break;
}
pollTimer.WaitOne(100);
}
}
}
private bool verifyUpdateFile(string filename, wsusUpdate toCheckAgainst)
{
using (SHA1 hasher = SHA1.Create())
{
using (FileStream stream = File.OpenRead(filename))
{
byte[] hashOnDisk = hasher.ComputeHash(stream);
if (hashOnDisk.SequenceEqual(toCheckAgainst.fileHashFromWSUS))
{
return(true);
}
return(false);
}
}
}
// public wsusUpdate parent { get; set; }
public wsusUpdate_cab(wsusUpdate parent, string absoluteLocation) : base()
{
this.parent = parent;
/* this.offlineCapable = false;
* this.description = "(none)";
* this.productName = "(none)";
* this.supportInfo = "(none)";
* try
* {
* string dismInfo = wsusUpdate.runAndGetStdout("dism", $"/online /get-packageinfo /packagepath:{absoluteLocation}");
* string[] dismInfoLines = dismInfo.Split('\n');
* foreach (string dismInfoLine in dismInfoLines)
* {
* Regex re = new Regex("^(?<name>.*?) : (?<value>.*)$");
* Match m = re.Match(dismInfoLine);
* if (m.Success == false)
* continue;
* string val = m.Groups["value"].ToString().Trim().ToLower();
* string name = m.Groups["name"].ToString().Trim().ToLower();
* switch (name)
* {
* case "completely offline capable":
* offlineCapable = parseBool(val, name);
* break;
* case "description":
* description = val;
* break;
* case "product name":
* productName = val;
* break;
* case "support information":
* supportInfo = val;
* break;
* }
* }
* }
* catch (Exception e)
* {
* // oh well
* }*/
}
public void bulkInsertFiles(wsusUpdate grandparent, file_wimInfo[] toInsertList)
{
using (SqlCommand cmd = _dbConn.CreateCommand())
{
string initialsql = $"Declare @testdata as FileTableType; ";
int thisbatchcnt = 0;
void flushBatch()
{
if (thisbatchcnt == 0)
{
return;
}
addParamWithValue(cmd, $"@parentFileHash", grandparent.fileHashFromWSUS);
addParamWithValue(cmd, $"@wimImageIndex", toInsertList[0].parent.wimImageIndex);
addParamWithValue(cmd, $"@wimImageSize", toInsertList[0].parent.wimImageSize);
addParamWithValue(cmd, $"@wimImageName", toInsertList[0].parent.wimImageName);
addParamWithValue(cmd, $"@wimImageDescription", toInsertList[0].parent.wimImageDescription);
cmd.CommandText +=
" exec insertfiles_wim @parentFileHash, @wimImageIndex, @wimImageSize, @wimImageName, @wimImageDescription, @testdata; ";
cmd.ExecuteNonQuery();
cmd.CommandText = initialsql;
thisbatchcnt = 0;
cmd.Parameters.Clear();
}
cmd.CommandText = initialsql;
foreach (file_wimInfo toInsert in toInsertList)
{
addFileToBatchCommand(toInsert.fileInfo, thisbatchcnt, cmd);
thisbatchcnt++;
if (thisbatchcnt == 80)
{
flushBatch();
}
}
flushBatch();
}
}
public wsusUpdate getWSUSFileByID(int ID)
{
using (SqlCommand cmd = _dbConn.CreateCommand())
{
cmd.CommandText =
"select * from wsusFile where id = @id";
addParamWithValue(cmd, "@id", ID);
using (SqlDataReader res = cmd.ExecuteReader())
{
if (res.Read() == false)
{
throw new Exception($"Couldn't find wsus file with id {ID}");
}
wsusUpdate toRet = new wsusUpdate(res);
toRet.dbID = ID;
return(toRet);
}
}
}
public static void importWSUSFiles()
{
using (updateDB db = new updateDB(connstr, dbName))
{
int updateIndexInBatch = 0;
wsusUpdate[] updates = new wsusUpdate[400];
foreach (IUpdate update in getUpdates())
{
foreach (IInstallableItem item in update.GetInstallableItems())
{
foreach (UpdateFile f in item.Files)
{
if (f.Type == FileType.Express ||
f.OriginUri.ToString().EndsWith(".txt"))
{
continue;
}
wsusUpdate upd = new wsusUpdate(f);
updates[updateIndexInBatch++] = upd;
if (updateIndexInBatch == updates.Length)
{
db.insert_noconcurrency(updates);
updateIndexInBatch = 0;
}
}
}
}
wsusUpdate[] updatesFinalBatch = new wsusUpdate[updateIndexInBatch];
Array.Copy(updates, updatesFinalBatch, updateIndexInBatch);
if (updateIndexInBatch == updates.Length)
{
db.insert_noconcurrency(updatesFinalBatch);
}
db.removeDuplicateWsusFiles();
}
}
public static updateInfo parse(wsusUpdate src, string absoluteLocation)
{
/*
* // Do an initial pass using the unix 'file' tool.
* string filesig = doGnuFileCheck(filename);
*
* switch (filesig)
* {
* case "Microsoft Cabinet archive data":
* return new wsusUpdate_cab(filename, deleteOnDestruction);
* // return new wsusUpdate_exe(filename, deleteOnDestruction);
* case "Windows imaging (WIM) image":
* return new wsusUpdate_wim(filename, deleteOnDestruction);
* }
*
* if (filesig.StartsWith("PE32"))
* return new wsusUpdate_exe(filename, deleteOnDestruction);
*/
string ext = Path.GetExtension(src.downloadURI.ToString()).ToLower();
switch (ext)
{
case ".cab":
return(new wsusUpdate_cab(new wsusUpdate(src), absoluteLocation));
case ".exe":
return(new wsusUpdate_exe(new wsusUpdate(src)));
case ".esd":
case ".wim":
return(new wsusUpdate_wim(new wsusUpdate(src)));
default:
throw new ArgumentException($"unknown file extension '{ext}'");
}
}
public void logError(wsusUpdate src, Exception e)
{
insertOrCreateWsusFileNoUpdate(src);
using (SqlCommand cmd = _dbConn.CreateCommand())
{
cmd.CommandText =
"Insert into errors (srcurl, exceptiontype, exceptionstring) values (@srcurl, @exceptiontype, @exceptionstring)";
if (src != null)
{
addParamWithValue(cmd, "@srcurl", src.downloadURI.ToString());
}
else
{
addParamWithValue(cmd, "@srcurl", DBNull.Value);
}
addParamWithValue(cmd, "@exceptiontype", e.GetType().ToString());
addParamWithValue(cmd, "@exceptionstring", e.ToString());
cmd.ExecuteNonQuery();
}
}
// public wsusUpdate parent { get; set; }
public wsusUpdate_wim(wsusUpdate parent)
{
this.parent = parent;
}
public void bulkInsertFiles(wsusUpdate parent, file_wimInfo[] wimFiles, bool isFinal)
{
parent.endTime = DateTime.Now;
batches_wim.Enqueue(new fileBatch_wim(parent, wimFiles, isFinal));
pollTimer.Set();
}
public void bulkInsertFiles(wsusUpdate parent, file[] files, bool isFinal = false)
{
batches.Enqueue(new fileBatch(parent, files, isFinal));
parent.endTime = DateTime.Now;
pollTimer.Set();
}
public fileBatch_wim(wsusUpdate parent, file_wimInfo[] files, bool isFinal)
{
Parent = parent;
Files = files;
IsFinal = isFinal;
}
public fileBatch(wsusUpdate parent, file[] files, bool isFinal)
{
Parent = parent;
Files = files;
IsFinal = isFinal;
}
public void insertOrCreateWsusFile(wsusUpdate toInsert)
{
insertOrCreate(toInsert, "wsusFile", "filename");
}
public void insert_noconcurrency(wsusUpdate toInsert)
{
insert_noconcurrency(new [] { toInsert }, "wsusFile");
}
protected override void threadmain()
{
WebRequest.DefaultWebProxy = null;
while (true)
{
if (updateReserver.allUpdatesExhausted())
{
if (exitTime)
{
break;
}
pollTimer.WaitOne(100);
continue;
}
// Keep a maximum of this many files on disk
if (outqueue.Count > maxFinishedDownloads)
{
pollTimer.WaitOne(100);
continue;
}
// Get an update to download from the input queue
if (updateReserver.tryDequeueUpdate(out wsusUpdate toDownload) == false)
{
continue;
}
currentlydownloading = toDownload;
string outputfile = null;
// If the file already exists in the archive or temp dir, use it from that location.
string tempfile = Path.Combine(Program.tempdir, toDownload.getTemporaryFilename());
if (File.Exists(tempfile))
{
outputfile = tempfile;
}
if (Program.archivedir != null)
{
string archivefile = Path.Combine(Program.archivedir, toDownload.getTemporaryFilename());
if (File.Exists(archivefile))
{
outputfile = archivefile;
}
}
if (File.Exists(outputfile))
{
// The file already exists. Let's check its length and hash, just to be sure it's what we want.
if (verifyUpdateFile(outputfile, toDownload) == false)
{
// It's corrupt, or partially downloaded. Delete it and we'll start again.
archiveThread.deleteWithRetry(outputfile, OnLogString);
outputfile = null;
}
}
// Now we should download the file if necessary, into the temporary file location.
while (outputfile == null)
{
try
{
using (WebClient client = new WebClient())
{
client.Proxy = null;
client.DownloadFile(toDownload.downloadURI, tempfile);
currentlydownloading = null;
outputfile = tempfile;
}
}
catch (WebException e)
{
logString(e, $"Download of {toDownload.downloadURI} to {outputfile} failed");
Thread.Sleep(TimeSpan.FromSeconds(10));
}
}
// Finally, pass the downloaded update off to the processing thread queue.
try
{
downloadedUpdate newlyDownloaded = new downloadedUpdate();
newlyDownloaded.absolutepath = outputfile;
newlyDownloaded.update = wsusUpdate.parse(toDownload, outputfile);
outqueue.Enqueue(newlyDownloaded);
downloadComplete.Set();
}
catch (Exception e)
{
logString(e, $"Processing of update from {toDownload.downloadURI} as {outputfile} failed");
}
}
}
public wsusUpdate startNextUpdate()
{
int toRetID;
using (SqlTransaction trans = _dbConn.BeginTransaction(IsolationLevel.Serializable))
{
using (SqlCommand cmd = _dbConn.CreateCommand())
{
cmd.Transaction = trans;
cmd.CommandText = "select 1 from wsusFile with (holdlock, tablockx)";
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = _dbConn.CreateCommand())
{
string orderstanza = "order by sizeBytes ";
if (Program.biggestfirst)
{
orderstanza += "desc";
}
else
{
orderstanza += "asc";
}
cmd.Transaction = trans;
cmd.CommandText = @"update E
set status = @hostname
output INSERTED.id
FROM
(
select top 1 id, status
from wsusFile
where status = 'QUEUED'
" + orderstanza + @"
) E
";
addParamWithValue(cmd, "hostname", getMachineHostname());
using (SqlDataReader reader = cmd.ExecuteReader())
{
if (reader.Read())
{
toRetID = (int)reader[0];
}
else
{
toRetID = int.MinValue;
}
}
}
trans.Commit();
}
if (toRetID == int.MinValue)
{
return(null);
}
Debug.WriteLine($"Thread {Thread.CurrentThread.ManagedThreadId} started update {toRetID}");
wsusUpdate toRet = getWSUSFileByID(toRetID);
return(toRet);
}
// public wsusUpdate parent { get; set; }
public wsusUpdate_exe(wsusUpdate parent) : base()
{
this.parent = parent;
}
public file_wimInfo(wsusUpdate grandparent, updateFile source, fileSource_wim parent)
{
fileInfo = new file(grandparent, source);
this.parent = parent;
}
public bulkDBInsert_file_wiminfo(wsusUpdate parent, asyncsqlthread db) : base(parent, db)
{
}