public bulkDBInsert(wsusUpdate parent, asyncsqlthread db) { _parent = parent; _db = db; filesInBatchSoFar = 0; files = new T[80]; }
public file(wsusUpdate parent, updateFile src) { this.sourceFilename = parent.filename; this.filename = src.filename; this.filelocation = src.filelocation; this.fileextension = src.fileextension; this.hash_sha256 = src.hash_sha256; this.size = (long)src.size; this.pe_sizeOfCode = toNullableBinary(src.pe_sizeOfCode); this.pe_timestamp = toNullableBinary(src.pe_timedatestamp); this.pe_magicType = toNullableBinary(src.pe_magicType); this.contents128b = src.contents128b; this.rsds_GUID = src.rsds_GUID; this.rsds_age = (int)src.rsds_age; this.rsds_filename = src.rsds_filename; this.authenticode_certfriendly = src.authenticode_certfriendly; this.authenticode_certsubj = src.authenticode_certsubj; this.FileDescription = src.FileDescription; this.FileVersion = src.FileVersion; this.ProductName = src.ProductName; this.ProductVersion = src.ProductVersion; this.Comments = src.Comments; this.CompanyName = src.CompanyName; }
public wsusUpdate(wsusUpdate src) { this.fileHashFromWSUS = src.fileHashFromWSUS; this.downloadURI = src.downloadURI; this.sizeBytes = src.sizeBytes; this.filename = getTemporaryFilename(); }
public void bulkInsertFiles(wsusUpdate grandparent, IEnumerable <file> toInsertList) { using (SqlCommand cmd = _dbConn.CreateCommand()) { string initialsql = $"Declare @testdata as FileTableType; "; cmd.CommandText = initialsql; int thisbatchcnt = 0; void flushBatch() { addParamWithValue(cmd, $"@parentFileHash", grandparent.fileHashFromWSUS); cmd.CommandText += " exec insertfiles @parentFileHash,@testdata; "; cmd.ExecuteNonQuery(); cmd.CommandText = $"Declare @testdata as FileTableType; "; thisbatchcnt = 0; cmd.Parameters.Clear(); } foreach (file toInsert in toInsertList) { addFileToBatchCommand(toInsert, thisbatchcnt, cmd); thisbatchcnt++; if (thisbatchcnt == 80) { flushBatch(); } } flushBatch(); } }
public fileSource_cab(wsusUpdate parent, wsusUpdate_cab src) { this.wsusFileID = parent.dbID.Value; this.offlineCapable = src.offlineCapable; this.description = src.description; this.productName = src.productName; this.supportInfo = src.supportInfo; }
public void insertOrCreateWsusFileNoUpdate(wsusUpdate toInsert) { if (toInsert.dbID.HasValue) { return; } insertOrCreateNoUpdate(toInsert, "wsusFile", new [] { "filename" }); }
public void insertOrCreateCabFileNoUpdate(wsusUpdate parent, wsusUpdate_cab metadata) { if (parent.dbID == null) { insertOrCreateWsusFileNoUpdate(parent); } insert_noconcurrency(new [] { new fileSource_cab(parent, metadata) }, "filesource_cab"); }
public bool tryDequeueUpdate(out wsusUpdate outputOrNull) { bool toRet = claimedUpdates.TryDequeue(out outputOrNull); if (toRet == false || outputOrNull == null) { // If we are stalling, ask the mail thread for more update IDs. pollTimer.Set(); } return(toRet); }
protected override void threadmain() { using (updateDB db = new updateDB(Program.connstr, Program.dbName)) { // Read anything we queued in a previous run foreach (var alreadyClaimed in db.getClaimedUpdate()) { claimedUpdates.Enqueue(alreadyClaimed); } // Make sure we have at least highWaterMark updates queued (if they are available) while (claimedUpdates.Count < highWaterMark) { wsusUpdate nextUpdate = db.startNextUpdate(); if (nextUpdate == null) { _allUpdatesExhausted = true; exitTime = true; break; } claimedUpdates.Enqueue(nextUpdate); } while (true) { // if we drop below the low water mark, queue updates until we get to the high water mark. if (claimedUpdates.Count < lowWaterMark) { while (claimedUpdates.Count < highWaterMark) { wsusUpdate nextUpdate = db.startNextUpdate(); if (nextUpdate == null) { _allUpdatesExhausted = true; exitTime = true; break; } claimedUpdates.Enqueue(nextUpdate); } } if (exitTime) { // TODO: should we unmark what we have in progress before exiting? break; } pollTimer.WaitOne(100); } } }
private bool verifyUpdateFile(string filename, wsusUpdate toCheckAgainst) { using (SHA1 hasher = SHA1.Create()) { using (FileStream stream = File.OpenRead(filename)) { byte[] hashOnDisk = hasher.ComputeHash(stream); if (hashOnDisk.SequenceEqual(toCheckAgainst.fileHashFromWSUS)) { return(true); } return(false); } } }
// public wsusUpdate parent { get; set; } public wsusUpdate_cab(wsusUpdate parent, string absoluteLocation) : base() { this.parent = parent; /* this.offlineCapable = false; * this.description = "(none)"; * this.productName = "(none)"; * this.supportInfo = "(none)"; * try * { * string dismInfo = wsusUpdate.runAndGetStdout("dism", $"/online /get-packageinfo /packagepath:{absoluteLocation}"); * string[] dismInfoLines = dismInfo.Split('\n'); * foreach (string dismInfoLine in dismInfoLines) * { * Regex re = new Regex("^(?<name>.*?) : (?<value>.*)$"); * Match m = re.Match(dismInfoLine); * if (m.Success == false) * continue; * string val = m.Groups["value"].ToString().Trim().ToLower(); * string name = m.Groups["name"].ToString().Trim().ToLower(); * switch (name) * { * case "completely offline capable": * offlineCapable = parseBool(val, name); * break; * case "description": * description = val; * break; * case "product name": * productName = val; * break; * case "support information": * supportInfo = val; * break; * } * } * } * catch (Exception e) * { * // oh well * }*/ }
public void bulkInsertFiles(wsusUpdate grandparent, file_wimInfo[] toInsertList) { using (SqlCommand cmd = _dbConn.CreateCommand()) { string initialsql = $"Declare @testdata as FileTableType; "; int thisbatchcnt = 0; void flushBatch() { if (thisbatchcnt == 0) { return; } addParamWithValue(cmd, $"@parentFileHash", grandparent.fileHashFromWSUS); addParamWithValue(cmd, $"@wimImageIndex", toInsertList[0].parent.wimImageIndex); addParamWithValue(cmd, $"@wimImageSize", toInsertList[0].parent.wimImageSize); addParamWithValue(cmd, $"@wimImageName", toInsertList[0].parent.wimImageName); addParamWithValue(cmd, $"@wimImageDescription", toInsertList[0].parent.wimImageDescription); cmd.CommandText += " exec insertfiles_wim @parentFileHash, @wimImageIndex, @wimImageSize, @wimImageName, @wimImageDescription, @testdata; "; cmd.ExecuteNonQuery(); cmd.CommandText = initialsql; thisbatchcnt = 0; cmd.Parameters.Clear(); } cmd.CommandText = initialsql; foreach (file_wimInfo toInsert in toInsertList) { addFileToBatchCommand(toInsert.fileInfo, thisbatchcnt, cmd); thisbatchcnt++; if (thisbatchcnt == 80) { flushBatch(); } } flushBatch(); } }
public wsusUpdate getWSUSFileByID(int ID) { using (SqlCommand cmd = _dbConn.CreateCommand()) { cmd.CommandText = "select * from wsusFile where id = @id"; addParamWithValue(cmd, "@id", ID); using (SqlDataReader res = cmd.ExecuteReader()) { if (res.Read() == false) { throw new Exception($"Couldn't find wsus file with id {ID}"); } wsusUpdate toRet = new wsusUpdate(res); toRet.dbID = ID; return(toRet); } } }
public static void importWSUSFiles() { using (updateDB db = new updateDB(connstr, dbName)) { int updateIndexInBatch = 0; wsusUpdate[] updates = new wsusUpdate[400]; foreach (IUpdate update in getUpdates()) { foreach (IInstallableItem item in update.GetInstallableItems()) { foreach (UpdateFile f in item.Files) { if (f.Type == FileType.Express || f.OriginUri.ToString().EndsWith(".txt")) { continue; } wsusUpdate upd = new wsusUpdate(f); updates[updateIndexInBatch++] = upd; if (updateIndexInBatch == updates.Length) { db.insert_noconcurrency(updates); updateIndexInBatch = 0; } } } } wsusUpdate[] updatesFinalBatch = new wsusUpdate[updateIndexInBatch]; Array.Copy(updates, updatesFinalBatch, updateIndexInBatch); if (updateIndexInBatch == updates.Length) { db.insert_noconcurrency(updatesFinalBatch); } db.removeDuplicateWsusFiles(); } }
public static updateInfo parse(wsusUpdate src, string absoluteLocation) { /* * // Do an initial pass using the unix 'file' tool. * string filesig = doGnuFileCheck(filename); * * switch (filesig) * { * case "Microsoft Cabinet archive data": * return new wsusUpdate_cab(filename, deleteOnDestruction); * // return new wsusUpdate_exe(filename, deleteOnDestruction); * case "Windows imaging (WIM) image": * return new wsusUpdate_wim(filename, deleteOnDestruction); * } * * if (filesig.StartsWith("PE32")) * return new wsusUpdate_exe(filename, deleteOnDestruction); */ string ext = Path.GetExtension(src.downloadURI.ToString()).ToLower(); switch (ext) { case ".cab": return(new wsusUpdate_cab(new wsusUpdate(src), absoluteLocation)); case ".exe": return(new wsusUpdate_exe(new wsusUpdate(src))); case ".esd": case ".wim": return(new wsusUpdate_wim(new wsusUpdate(src))); default: throw new ArgumentException($"unknown file extension '{ext}'"); } }
public void logError(wsusUpdate src, Exception e) { insertOrCreateWsusFileNoUpdate(src); using (SqlCommand cmd = _dbConn.CreateCommand()) { cmd.CommandText = "Insert into errors (srcurl, exceptiontype, exceptionstring) values (@srcurl, @exceptiontype, @exceptionstring)"; if (src != null) { addParamWithValue(cmd, "@srcurl", src.downloadURI.ToString()); } else { addParamWithValue(cmd, "@srcurl", DBNull.Value); } addParamWithValue(cmd, "@exceptiontype", e.GetType().ToString()); addParamWithValue(cmd, "@exceptionstring", e.ToString()); cmd.ExecuteNonQuery(); } }
// public wsusUpdate parent { get; set; } public wsusUpdate_wim(wsusUpdate parent) { this.parent = parent; }
public void bulkInsertFiles(wsusUpdate parent, file_wimInfo[] wimFiles, bool isFinal) { parent.endTime = DateTime.Now; batches_wim.Enqueue(new fileBatch_wim(parent, wimFiles, isFinal)); pollTimer.Set(); }
public void bulkInsertFiles(wsusUpdate parent, file[] files, bool isFinal = false) { batches.Enqueue(new fileBatch(parent, files, isFinal)); parent.endTime = DateTime.Now; pollTimer.Set(); }
public fileBatch_wim(wsusUpdate parent, file_wimInfo[] files, bool isFinal) { Parent = parent; Files = files; IsFinal = isFinal; }
public fileBatch(wsusUpdate parent, file[] files, bool isFinal) { Parent = parent; Files = files; IsFinal = isFinal; }
public void insertOrCreateWsusFile(wsusUpdate toInsert) { insertOrCreate(toInsert, "wsusFile", "filename"); }
public void insert_noconcurrency(wsusUpdate toInsert) { insert_noconcurrency(new [] { toInsert }, "wsusFile"); }
protected override void threadmain() { WebRequest.DefaultWebProxy = null; while (true) { if (updateReserver.allUpdatesExhausted()) { if (exitTime) { break; } pollTimer.WaitOne(100); continue; } // Keep a maximum of this many files on disk if (outqueue.Count > maxFinishedDownloads) { pollTimer.WaitOne(100); continue; } // Get an update to download from the input queue if (updateReserver.tryDequeueUpdate(out wsusUpdate toDownload) == false) { continue; } currentlydownloading = toDownload; string outputfile = null; // If the file already exists in the archive or temp dir, use it from that location. string tempfile = Path.Combine(Program.tempdir, toDownload.getTemporaryFilename()); if (File.Exists(tempfile)) { outputfile = tempfile; } if (Program.archivedir != null) { string archivefile = Path.Combine(Program.archivedir, toDownload.getTemporaryFilename()); if (File.Exists(archivefile)) { outputfile = archivefile; } } if (File.Exists(outputfile)) { // The file already exists. Let's check its length and hash, just to be sure it's what we want. if (verifyUpdateFile(outputfile, toDownload) == false) { // It's corrupt, or partially downloaded. Delete it and we'll start again. archiveThread.deleteWithRetry(outputfile, OnLogString); outputfile = null; } } // Now we should download the file if necessary, into the temporary file location. while (outputfile == null) { try { using (WebClient client = new WebClient()) { client.Proxy = null; client.DownloadFile(toDownload.downloadURI, tempfile); currentlydownloading = null; outputfile = tempfile; } } catch (WebException e) { logString(e, $"Download of {toDownload.downloadURI} to {outputfile} failed"); Thread.Sleep(TimeSpan.FromSeconds(10)); } } // Finally, pass the downloaded update off to the processing thread queue. try { downloadedUpdate newlyDownloaded = new downloadedUpdate(); newlyDownloaded.absolutepath = outputfile; newlyDownloaded.update = wsusUpdate.parse(toDownload, outputfile); outqueue.Enqueue(newlyDownloaded); downloadComplete.Set(); } catch (Exception e) { logString(e, $"Processing of update from {toDownload.downloadURI} as {outputfile} failed"); } } }
public wsusUpdate startNextUpdate() { int toRetID; using (SqlTransaction trans = _dbConn.BeginTransaction(IsolationLevel.Serializable)) { using (SqlCommand cmd = _dbConn.CreateCommand()) { cmd.Transaction = trans; cmd.CommandText = "select 1 from wsusFile with (holdlock, tablockx)"; cmd.ExecuteNonQuery(); } using (SqlCommand cmd = _dbConn.CreateCommand()) { string orderstanza = "order by sizeBytes "; if (Program.biggestfirst) { orderstanza += "desc"; } else { orderstanza += "asc"; } cmd.Transaction = trans; cmd.CommandText = @"update E set status = @hostname output INSERTED.id FROM ( select top 1 id, status from wsusFile where status = 'QUEUED' " + orderstanza + @" ) E "; addParamWithValue(cmd, "hostname", getMachineHostname()); using (SqlDataReader reader = cmd.ExecuteReader()) { if (reader.Read()) { toRetID = (int)reader[0]; } else { toRetID = int.MinValue; } } } trans.Commit(); } if (toRetID == int.MinValue) { return(null); } Debug.WriteLine($"Thread {Thread.CurrentThread.ManagedThreadId} started update {toRetID}"); wsusUpdate toRet = getWSUSFileByID(toRetID); return(toRet); }
// public wsusUpdate parent { get; set; } public wsusUpdate_exe(wsusUpdate parent) : base() { this.parent = parent; }
public file_wimInfo(wsusUpdate grandparent, updateFile source, fileSource_wim parent) { fileInfo = new file(grandparent, source); this.parent = parent; }
public bulkDBInsert_file_wiminfo(wsusUpdate parent, asyncsqlthread db) : base(parent, db) { }