public static void ClamScanAllFiles() { UpdateProgress2?.Invoke("Asking database for files", null, 0, 0); #if DEBUG stopwatch.Restart(); #endif if (!dbCore.DbOps.GetNotAvFiles(out List <DbFile> files)) { Failed?.Invoke("Could not get files from database."); } #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.ClamScanAllFiles(): Took {0} seconds to get files from database", stopwatch.Elapsed.TotalSeconds); stopwatch.Restart(); #endif int counter = 0; foreach (DbFile file in files) { UpdateProgress2?.Invoke($"Scanning file {counter} of {files.Count}", null, counter, files.Count); ClamScanFileFromRepo(file); counter++; } #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.ClamScanAllFiles(): Took {0} seconds scan all pending files", stopwatch.Elapsed.TotalSeconds); #endif Finished?.Invoke(); }
public static void CheckDbForFiles() { try { long counter = 0; #if DEBUG stopwatch.Restart(); #endif Dictionary <string, DbOsFile> knownFiles = new Dictionary <string, DbOsFile>(); bool unknownFile = false; foreach (KeyValuePair <string, DbOsFile> kvp in Context.Hashes) { // Empty file with size zero if (kvp.Value.Sha256 == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") { AddFileForOS(kvp.Key, kvp.Value.Sha256, true, kvp.Value.Crack); counter++; continue; } UpdateProgress?.Invoke(null, "Checking files in database", counter, Context.Hashes.Count); AddFileForOS?.Invoke(kvp.Key, kvp.Value.Sha256, dbCore.DbOps.ExistsFile(kvp.Value.Sha256), kvp.Value.Crack); if (dbCore.DbOps.ExistsFile(kvp.Value.Sha256)) { counter++; knownFiles.Add(kvp.Key, kvp.Value); } else { unknownFile = true; } } #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.CheckDbForFiles(): Took {0} seconds to checks for file knowledge in the DB", stopwatch.Elapsed.TotalSeconds); stopwatch.Restart(); #endif if (knownFiles.Count == 0 || unknownFile) { Finished?.Invoke(); return; } UpdateProgress?.Invoke(null, "Retrieving OSes from database", counter, Context.Hashes.Count); dbCore.DbOps.GetAllOSes(out List <DbEntry> oses); #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.CheckDbForFiles(): Took {0} seconds get all OSes from DB", stopwatch.Elapsed.TotalSeconds); #endif if (oses != null && oses.Count > 0) { DbEntry[] osesArray = new DbEntry[oses.Count]; oses.CopyTo(osesArray); long osCounter = 0; #if DEBUG stopwatch.Restart(); #endif foreach (DbEntry os in osesArray) { UpdateProgress?.Invoke(null, $"Check OS id {os.Id}", osCounter, osesArray.Length); counter = 0; foreach (KeyValuePair <string, DbOsFile> kvp in knownFiles) { UpdateProgress2?.Invoke(null, $"Checking for file {kvp.Value.Path}", counter, knownFiles.Count); if (!dbCore.DbOps.ExistsFileInOs(kvp.Value.Sha256, os.Id)) { if (oses.Contains(os)) { oses.Remove(os); } // If one file is missing, the rest don't matter break; } counter++; } if (oses.Count == 0) { break; // No OSes left } } #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.CheckDbForFiles(): Took {0} seconds correlate all files with all known OSes", stopwatch.Elapsed.TotalSeconds); #endif } if (AddOS != null) { foreach (DbEntry os in oses) { string destination = Path.Combine(Settings.Current.RepositoryPath, os.Mdid[0].ToString(), os.Mdid[1].ToString(), os.Mdid[2].ToString(), os.Mdid[3].ToString(), os.Mdid[4].ToString(), os.Mdid) + ".zip"; AddOS?.Invoke(os); } } Finished?.Invoke(); } catch (ThreadAbortException) { } catch (Exception ex) { if (Debugger.IsAttached) { throw; } Failed?.Invoke($"Exception {ex.Message}\n{ex.InnerException}"); #if DEBUG Console.WriteLine("Exception {0}\n{1}", ex.Message, ex.InnerException); #endif } }
public static void CompressFiles() { try { if (string.IsNullOrWhiteSpace(Context.DbInfo.Developer)) { Failed?.Invoke("Developer cannot be empty"); return; } if (string.IsNullOrWhiteSpace(Context.DbInfo.Product)) { Failed?.Invoke("Product cannot be empty"); return; } if (string.IsNullOrWhiteSpace(Context.DbInfo.Version)) { Failed?.Invoke("Version cannot be empty"); return; } string destinationFolder = ""; destinationFolder = Path.Combine(destinationFolder, Context.DbInfo.Developer); destinationFolder = Path.Combine(destinationFolder, Context.DbInfo.Product); destinationFolder = Path.Combine(destinationFolder, Context.DbInfo.Version); if (!string.IsNullOrWhiteSpace(Context.DbInfo.Languages)) { destinationFolder = Path.Combine(destinationFolder, Context.DbInfo.Languages); } if (!string.IsNullOrWhiteSpace(Context.DbInfo.Architecture)) { destinationFolder = Path.Combine(destinationFolder, Context.DbInfo.Architecture); } if (Context.DbInfo.Oem) { destinationFolder = Path.Combine(destinationFolder, "oem"); } if (!string.IsNullOrWhiteSpace(Context.DbInfo.Machine)) { destinationFolder = Path.Combine(destinationFolder, "for " + Context.DbInfo.Machine); } string destinationFile = ""; if (!string.IsNullOrWhiteSpace(Context.DbInfo.Format)) { destinationFile += "[" + Context.DbInfo.Format + "]"; } if (Context.DbInfo.Files) { if (destinationFile != "") { destinationFile += "_"; } destinationFile += "files"; } if (Context.DbInfo.Netinstall) { if (destinationFile != "") { destinationFile += "_"; } destinationFile += "netinstall"; } if (Context.DbInfo.Source) { if (destinationFile != "") { destinationFile += "_"; } destinationFile += "source"; } if (Context.DbInfo.Update) { if (destinationFile != "") { destinationFile += "_"; } destinationFile += "update"; } if (Context.DbInfo.Upgrade) { if (destinationFile != "") { destinationFile += "_"; } destinationFile += "upgrade"; } if (!string.IsNullOrWhiteSpace(Context.DbInfo.Description)) { if (destinationFile != "") { destinationFile += "_"; } destinationFile += Context.DbInfo.Description; } else if (destinationFile == "") { destinationFile = "archive"; } string destination = Path.Combine(destinationFolder, destinationFile) + ".zip"; Md5Context md5 = new Md5Context(); md5.Init(); byte[] tmp; string mdid = md5.Data(Encoding.UTF8.GetBytes(destination), out tmp); Console.WriteLine("MDID: {0}", mdid); if (dbCore.DbOps.ExistsOs(mdid)) { if (File.Exists(destination)) { Failed?.Invoke("OS already exists."); return; } Failed?.Invoke("OS already exists in the database but not in the repository, check for inconsistencies."); return; } if (File.Exists(destination)) { Failed?.Invoke("OS already exists in the repository but not in the database, check for inconsistencies."); return; } Context.DbInfo.Mdid = mdid; string filesPath; if (!string.IsNullOrEmpty(Context.TmpFolder) && Directory.Exists(Context.TmpFolder)) { filesPath = Context.TmpFolder; } else { filesPath = Context.Path; } string extension = null; switch (Settings.Current.CompressionAlgorithm) { case AlgoEnum.GZip: extension = ".gz"; break; case AlgoEnum.BZip2: extension = ".bz2"; break; case AlgoEnum.LZMA: extension = ".lzma"; break; case AlgoEnum.LZip: extension = ".lz"; break; } long totalSize = 0, currentSize = 0; foreach (KeyValuePair <string, DbOsFile> file in Context.Hashes) { totalSize += file.Value.Length; } #if DEBUG stopwatch.Restart(); #endif foreach (KeyValuePair <string, DbOsFile> file in Context.Hashes) { UpdateProgress?.Invoke("Compressing...", file.Value.Path, currentSize, totalSize); destinationFolder = Path.Combine(Settings.Current.RepositoryPath, file.Value.Sha256[0].ToString(), file.Value.Sha256[1].ToString(), file.Value.Sha256[2].ToString(), file.Value.Sha256[3].ToString(), file.Value.Sha256[4].ToString()); Directory.CreateDirectory(destinationFolder); destinationFile = Path.Combine(destinationFolder, file.Value.Sha256 + extension); if (!File.Exists(destinationFile)) { FileStream inFs = new FileStream(Path.Combine(filesPath, file.Value.Path), FileMode.Open, FileAccess.Read); FileStream outFs = new FileStream(destinationFile, FileMode.CreateNew, FileAccess.Write); Stream zStream = null; switch (Settings.Current.CompressionAlgorithm) { case AlgoEnum.GZip: zStream = new GZipStream(outFs, CompressionMode.Compress, CompressionLevel.BestCompression); break; case AlgoEnum.BZip2: zStream = new BZip2Stream(outFs, CompressionMode.Compress); break; case AlgoEnum.LZMA: zStream = new LzmaStream(new LzmaEncoderProperties(), false, outFs); outFs.Write(((LzmaStream)zStream).Properties, 0, ((LzmaStream)zStream).Properties.Length); outFs.Write(BitConverter.GetBytes(inFs.Length), 0, 8); break; case AlgoEnum.LZip: zStream = new LZipStream(outFs, CompressionMode.Compress); break; } byte[] buffer = new byte[BUFFER_SIZE]; while (inFs.Position + BUFFER_SIZE <= inFs.Length) { UpdateProgress2?.Invoke($"{inFs.Position / (double)inFs.Length:P}", $"{inFs.Position} / {inFs.Length} bytes", inFs.Position, inFs.Length); UpdateProgress?.Invoke("Compressing...", file.Value.Path, currentSize, totalSize); inFs.Read(buffer, 0, buffer.Length); zStream.Write(buffer, 0, buffer.Length); currentSize += buffer.Length; } buffer = new byte[inFs.Length - inFs.Position]; UpdateProgress2?.Invoke($"{inFs.Position / (double)inFs.Length:P}", $"{inFs.Position} / {inFs.Length} bytes", inFs.Position, inFs.Length); UpdateProgress?.Invoke("Compressing...", file.Value.Path, currentSize, totalSize); inFs.Read(buffer, 0, buffer.Length); zStream.Write(buffer, 0, buffer.Length); currentSize += buffer.Length; UpdateProgress2?.Invoke($"{inFs.Length / (double)inFs.Length:P}", "Finishing...", inFs.Length, inFs.Length); inFs.Close(); zStream.Close(); outFs.Dispose(); } else { currentSize += file.Value.Length; } } #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.CompressFiles(): Took {0} seconds to compress files", stopwatch.Elapsed.TotalSeconds); #endif if (Context.Metadata != null) { MemoryStream xms = new MemoryStream(); XmlSerializer xs = new XmlSerializer(typeof(CICMMetadataType)); xs.Serialize(xms, Context.Metadata); xms.Position = 0; JsonSerializer js = new JsonSerializer { Formatting = Formatting.Indented, NullValueHandling = NullValueHandling.Ignore }; MemoryStream jms = new MemoryStream(); StreamWriter sw = new StreamWriter(jms, Encoding.UTF8, 1048576, true); js.Serialize(sw, Context.Metadata, typeof(CICMMetadataType)); sw.Close(); jms.Position = 0; destinationFolder = Path.Combine(Settings.Current.RepositoryPath, "metadata", mdid[0].ToString(), mdid[1].ToString(), mdid[2].ToString(), mdid[3].ToString(), mdid[4].ToString()); Directory.CreateDirectory(destinationFolder); FileStream xfs = new FileStream(Path.Combine(destinationFolder, mdid + ".xml"), FileMode.CreateNew, FileAccess.Write); xms.CopyTo(xfs); xfs.Close(); FileStream jfs = new FileStream(Path.Combine(destinationFolder, mdid + ".json"), FileMode.CreateNew, FileAccess.Write); jms.CopyTo(jfs); jfs.Close(); xms.Position = 0; jms.Position = 0; } FinishedWithText?.Invoke($"Correctly added operating system with MDID {mdid}"); } catch (ThreadAbortException) { } catch (Exception ex) { if (Debugger.IsAttached) { throw; } Failed?.Invoke($"Exception {ex.Message}\n{ex.InnerException}"); #if DEBUG Console.WriteLine("Exception {0}\n{1}", ex.Message, ex.InnerException); #endif } }
public static void CheckDbForFiles() { try { long counter = 0; #if DEBUG stopwatch.Restart(); #endif Dictionary <string, DbAppFile> knownFiles = new Dictionary <string, DbAppFile>(); bool unknownFile = false; foreach (KeyValuePair <string, DbAppFile> kvp in Context.Hashes) { // Empty file with size zero if (kvp.Value.Sha256 == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") { AddFileForApp(kvp.Key, kvp.Value.Sha256, true, kvp.Value.Crack); counter++; continue; } UpdateProgress?.Invoke(null, "Checking files in database", counter, Context.Hashes.Count); AddFileForApp?.Invoke(kvp.Key, kvp.Value.Sha256, dbCore.DbOps.ExistsFile(kvp.Value.Sha256), kvp.Value.Crack); if (dbCore.DbOps.ExistsFile(kvp.Value.Sha256)) { counter++; knownFiles.Add(kvp.Key, kvp.Value); } else { unknownFile = true; } } #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.CheckDbForFiles(): Took {0} seconds to checks for file knowledge in the DB", stopwatch.Elapsed.TotalSeconds); stopwatch.Restart(); #endif if (knownFiles.Count == 0 || unknownFile) { Finished?.Invoke(); return; } UpdateProgress?.Invoke(null, "Retrieving apps from database", counter, Context.Hashes.Count); dbCore.DbOps.GetAllApps(out List <DbEntry> apps); #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.CheckDbForFiles(): Took {0} seconds get all apps from DB", stopwatch.Elapsed.TotalSeconds); #endif if (apps != null && apps.Count > 0) { DbEntry[] appsArray = new DbEntry[apps.Count]; apps.CopyTo(appsArray); long appCounter = 0; #if DEBUG stopwatch.Restart(); #endif foreach (DbEntry app in appsArray) { UpdateProgress?.Invoke(null, $"Check application id {app.Id}", appCounter, appsArray.Length); counter = 0; foreach (KeyValuePair <string, DbAppFile> kvp in knownFiles) { UpdateProgress2?.Invoke(null, $"Checking for file {kvp.Value.Path}", counter, knownFiles.Count); if (!dbCore.DbOps.ExistsFileInApp(kvp.Value.Sha256, app.Id)) { if (apps.Contains(app)) { apps.Remove(app); } // If one file is missing, the rest don't matter break; } counter++; } if (apps.Count == 0) { break; // No apps left } } #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.CheckDbForFiles(): Took {0} seconds correlate all files with all known applications", stopwatch.Elapsed.TotalSeconds); #endif } if (AddApp != null) { foreach (DbEntry app in apps) { AddApp?.Invoke(app); } } Finished?.Invoke(); } catch (ThreadAbortException) {} catch (Exception ex) { if (Debugger.IsAttached) { throw; } Failed?.Invoke($"Exception {ex.Message}\n{ex.InnerException}"); #if DEBUG Console.WriteLine("Exception {0}\n{1}", ex.Message, ex.InnerException); #endif } }