public static async Task UploadPackagedInis(IEnumerable <IndexedArchive> archives) { archives = archives.ToArray(); // defensive copy Utils.Log($"Packaging {archives.Count()} inis"); try { await using var ms = new MemoryStream(); using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true)) { foreach (var archive in archives) { var state = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(archive.IniData)); var entry = z.CreateEntry(Path.GetFileName(archive.Name)); await using var os = entry.Open(); await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", state.GetMetaIni()))); } } var webClient = new WebClient(); await webClient.UploadDataTaskAsync($"https://{Consts.WabbajackCacheHostname}/indexed_files/notify", "POST", ms.ToArray()); } catch (Exception ex) { Utils.Log(ex.ToString()); } }
public override async ValueTask <Directive> Run(RawSourceFile source) { var filename = Path.GetFileName(source.Path); var gameFile = Path.Combine(_mo2Compiler.GamePath, "Data", filename); if (!Consts.GameESMs.Contains(filename) || !source.Path.StartsWith("mods\\") || !File.Exists(gameFile)) { return(null); } Utils.Log( $"An ESM named {filename} was found in a mod that shares a name with one of the core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched"); var result = source.EvolveTo <CleanedESM>(); result.SourceESMHash = _compiler.VFS.Index.ByRootPath[gameFile].Hash; Utils.Status($"Generating patch of {filename}"); await using (var ms = new MemoryStream()) { await Utils.CreatePatch(File.ReadAllBytes(gameFile), File.ReadAllBytes(source.AbsolutePath), ms); var data = ms.ToArray(); result.SourceDataID = _compiler.IncludeFile(data); Utils.Log($"Generated a {data.Length} byte patch for {filename}"); } return(result); }
public object Convert(object value, Type targetType, object parameter, CultureInfo culture) { string _path = value as string; if (File.Exists(_path)) { if (_path.Length > 240) // 规避文件名和路径名过长问题 { if (!Directory.Exists("./temp")) { Directory.CreateDirectory("./temp"); } FileInfo fi = new FileInfo(_path); string _tmp = Path.GetFileName(_path); if (!File.Exists("./temp/" + _tmp)) { fi.CopyTo("./temp/" + _tmp, true); } return(new BitmapImage(new Uri(AppDomain.CurrentDomain.BaseDirectory + @"./temp/" + _tmp, UriKind.RelativeOrAbsolute))); } else { return(new BitmapImage(new Uri(_path, UriKind.RelativeOrAbsolute))); } } else { return(null); } }
internal static void QueueUploadOfMissingPDFs(Library library, List <PDFDocument> pdf_documents) { // Get a list of all the documents in the Intranet library List <string> existing_pdfs_full = IntranetLibraryTools.GetListOfDocumentsInLibrary(library.WebLibraryDetail.IntranetPath); HashSet <string> existing_pdfs = new HashSet <string>(); foreach (string existing_pdf in existing_pdfs_full) { existing_pdfs.Add(Path.GetFileName(existing_pdf)); } foreach (PDFDocument pdf_document in pdf_documents) { bool deleted = pdf_document.Deleted; // Try to upload all files that we have if (!deleted && pdf_document.DocumentExists) { string filename_full = PDFDocumentFileLocations.DocumentPath(library, pdf_document.Fingerprint, "pdf"); string filename_short = Path.GetFileName(filename_full); if (!existing_pdfs.Contains(filename_short)) { SyncQueues.Instance.QueuePut(pdf_document.Fingerprint, library); } } } }
private static string ConvertNormalFilenameToZoteroFilename(string bibtex_base_filename, string filename) { string bibtex_base_path = Path.GetDirectoryName(bibtex_base_filename); string file = Path.GetFileName(filename); string path = filename; /// Try shrink it to a relative path if (path.StartsWith(bibtex_base_path)) { path = path.Substring(bibtex_base_path.Length + 1); } else { Logging.Warn("Unable to convert path to Zotero form: {0}", path); } // Replace the slashes path = path.Replace('\\', '/'); // Concatenate string export_string = "" + file + ":" + path + ":" + "application/pdf"; return(export_string); }
private void ChooseCSLFile(CSLFileSource csl_file_source) { OpenFileDialog dialog = new OpenFileDialog(); dialog.Filter = "CSL style files|*.csl" + "|" + "All files|*.*"; dialog.CheckFileExists = true; dialog.Multiselect = false; string filename = null; string directory = null; // Decide what we want to use as the CSL file location if (CSLFileSource.TEXTBOX == csl_file_source) { try { CorrectStyleFilenameForNewDirectoryLocation(); directory = Path.GetDirectoryName(ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile); filename = Path.GetFileName(ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile); // If the directory no longer exists, kill our memory of it. We will pick the default directory again. if (!Directory.Exists(directory)) { directory = null; } } catch { } } if (null == directory) { directory = PDFDocumentCitingTools.BASE_STYLE_DIRECTORY; } // Set the dialog defaults dialog.FileName = filename; dialog.InitialDirectory = directory; // Get the user response if (true == dialog.ShowDialog()) { ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile = dialog.FileName; ConfigurationManager.Instance.ConfigurationRecord_Bindable.NotifyPropertyChanged(() => ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile); // Check if it is a dependent style string style_xml_filename = ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile; string root_style_filename = DependentStyleDetector.GetRootStyleFilename(style_xml_filename); if (root_style_filename != style_xml_filename) { if (null != root_style_filename) { MessageBoxes.Info("This style is dependent on another style, so InCite will be using this file instead:\n" + root_style_filename); } else { MessageBoxes.Info("This style is dependent on another style that is not available on your computer. Please download it before proceeding."); } } } }
/// <summary> /// MurMur3 hashes the file pointed to by this string /// </summary> /// <param name="file"></param> /// <returns></returns> public static string FileSHA256(this string file) { var sha = new SHA256Managed(); using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write)) { using (var i = File.OpenRead(file)) i.CopyToWithStatus(new FileInfo(file).Length, o, $"Hashing {Path.GetFileName(file)}"); } return(sha.Hash.ToBase64()); }
public static StreamReader openStreamReaderWithLocalCheck(String filename) { if (File.Exists(filename)) { return(new StreamReader(filename)); } else { return(new StreamReader(Path.GetFileName(filename))); } }
internal static void DaemonPut(Library library, string fingerprint) { string filename_full = PDFDocumentFileLocations.DocumentPath(library, fingerprint, "pdf"); string filename_short = Path.GetFileName(filename_full); string pdf_path = IntranetLibraryTools.GetLibraryPDFPath(library.WebLibraryDetail.IntranetPath, filename_short); DirectoryTools.CreateDirectory(Path.GetDirectoryName(pdf_path)); Logging.Info("+Copying up {0}", fingerprint); File.Copy(filename_full, pdf_path); Logging.Info("-Copying up {0}", fingerprint); }
private void UpdateArchive(VirtualFile f) { if (!f.IsStaged) { throw new InvalidDataException("Can't analyze an unstaged file"); } var tmp_dir = Path.Combine(_stagedRoot, Guid.NewGuid().ToString()); Utils.Status($"Extracting Archive {Path.GetFileName(f.StagedPath)}"); FileExtractor.ExtractAll(f.StagedPath, tmp_dir); Utils.Status($"Updating Archive {Path.GetFileName(f.StagedPath)}"); var entries = Directory.EnumerateFiles(tmp_dir, "*", SearchOption.AllDirectories) .Select(path => path.RelativeTo(tmp_dir)); var new_files = entries.Select(e => { var new_path = new string[f.Paths.Length + 1]; f.Paths.CopyTo(new_path, 0); new_path[f.Paths.Length] = e; var nf = new VirtualFile { Paths = new_path }; nf._stagedPath = Path.Combine(tmp_dir, e); Add(nf); return(nf); }).ToList(); // Analyze them new_files.PMap(file => { Utils.Status($"Analyzing {Path.GetFileName(file.StagedPath)}"); file.Analyze(); }); // Recurse into any archives in this archive new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(file)); f.FinishedIndexing = true; if (!_isSyncing) { SyncToDisk(); } Utils.Status("Cleaning Directory"); DeleteDirectory(tmp_dir); }
private void BackupFile(string file) { var path = Path.Combine(this._basePath, "BannerLordLauncher Backups"); try { if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } } catch (Exception e) { this.Log().Error(e); return; } if (!File.Exists(file)) { return; } var ext = Path.GetExtension(file); var i = 0; var newFile = Path.ChangeExtension(file, $"{ext}.{i:D3}"); Debug.Assert(newFile != null, nameof(newFile) + " != null"); newFile = Path.Combine(path, Path.GetFileName(newFile)); while (File.Exists(newFile)) { i++; newFile = Path.ChangeExtension(file, $"{ext}.{i:D3}"); Debug.Assert(newFile != null, nameof(newFile) + " != null"); newFile = Path.Combine(path, Path.GetFileName(newFile)); } if (i > 999) { return; } try { Debug.Assert(file != null, nameof(file) + " != null"); File.Move(file, newFile); } catch (Exception e) { this.Log().Error(e); } }
private static bool SetBrowserEmulationVersion(BrowserEmulationVersion browserEmulationVersion) { bool result; result = false; try { RegistryKey key; key = Registry.CurrentUser.OpenSubKey(BrowserEmulationKey, true); if (key != null) { string programName; programName = Path.GetFileName(Environment.GetCommandLineArgs()[0]); if (browserEmulationVersion != BrowserEmulationVersion.Default) { // if it's a valid value, update or create the value key.SetValue(programName, (int)browserEmulationVersion, RegistryValueKind.DWord); logger.Warn("SETTING REGISTRY:{0}-{1}-{2}-{3}", key.Name, programName, (int)browserEmulationVersion, RegistryValueKind.DWord.ToString()); } else { // otherwise, remove the existing value key.DeleteValue(programName, false); logger.Warn("DELETING REGISTRY KEY:{0}-{1}", key.Name, programName); } result = true; } } catch (SecurityException se) { // The user does not have the permissions required to read from the registry key. logger.Error(se); } catch (UnauthorizedAccessException uae) { // The user does not have the necessary registry rights. logger.Error(uae); } return(result); }
static void AddAppsToTar(Stream tar, string apps_directory, bool conventions) { string[] directories = Directory.GetDirectories(apps_directory); foreach (string directory in directories) { var app = Path.GetFileName(directory); Console.WriteLine($"[+] App: {app}"); // manifest comes first var fManifest = Path.Combine(directory, "_manifest"); using (var fin_manifest = File.OpenRead(fManifest)) { // AddFileToTarRaw(tarOutputStream, fin_manifest, new FileInfo(fManifest), $"apps/{app}/_manifest"); ABTar.WriteTarFile(app, "", "_manifest", fin_manifest, tar, out var _); } // write other files foreach (var directory2 in Directory.GetDirectories(directory)) { var domain = Path.GetFileName(directory2); if (conventions) { if (AndroidBackupConventions.TryGetValue(domain, out var dir2Real)) { Console.WriteLine("[+] {dir2} -> {dir2Real}"); domain = dir2Real; } else if (AndroidBackupConventions.ContainsValue(domain)) { // name accepted } else { dir2Real = $"{AndroidBackupWildcardDir}/{domain}"; Console.WriteLine("[!] {dir2} -> {dir2Real}"); domain = dir2Real; } } Console.WriteLine($"[+] - {domain}"); AddDirToTar(tar, directory2, app, domain, "", false); } } ABTar.FinishTar(tar); }
internal bool SetLabel() { bool isFileReadOnly = Utils.IsReadOnly(GrepResult); string basePath = string.IsNullOrWhiteSpace(searchFolderPath) ? string.Empty : searchFolderPath.TrimEnd('\\'); string displayedName = Path.GetFileName(GrepResult.FileNameDisplayed); if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowFilePathInResults) && GrepResult.FileNameDisplayed.Contains(basePath, StringComparison.CurrentCultureIgnoreCase)) { if (!string.IsNullOrWhiteSpace(basePath)) { displayedName = GrepResult.FileNameDisplayed.Substring(basePath.Length + 1).TrimStart('\\'); } else { displayedName = GrepResult.FileNameDisplayed; } } if (!string.IsNullOrWhiteSpace(GrepResult.AdditionalInformation)) { displayedName += " " + GrepResult.AdditionalInformation + " "; } int matchCount = GrepResult.Matches == null ? 0 : GrepResult.Matches.Count; if (matchCount > 0) { if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowVerboseMatchCount) && !GrepResult.IsHexFile) { var lineCount = GrepResult.Matches.Where(r => r.LineNumber > 0) .Select(r => r.LineNumber).Distinct().Count(); displayedName = TranslationSource.Format(Resources.Main_ResultList_CountMatchesOnLines, displayedName, matchCount, lineCount); } else { displayedName = string.Format(Resources.Main_ResultList_CountMatches, displayedName, matchCount); } } if (isFileReadOnly) { displayedName = displayedName + " " + Resources.Main_ResultList_ReadOnly; } Label = displayedName; return(isFileReadOnly); }
internal static void DaemonGet(Library library, string fingerprint) { string filename_full = PDFDocumentFileLocations.DocumentPath(library, fingerprint, "pdf"); string filename_short = Path.GetFileName(filename_full); string pdf_path = IntranetLibraryTools.GetLibraryPDFPath(library.WebLibraryDetail.IntranetPath, filename_short); DirectoryTools.CreateDirectory(Path.GetDirectoryName(filename_full)); Logging.Info("+Copying down {0}", fingerprint); File.Copy(pdf_path, filename_full); Logging.Info("-Copying down {0}", fingerprint); // Write the audit if (true) { string audit_filename = IntranetLibraryTools.GetLibraryAuditFilename(library.WebLibraryDetail.IntranetPath); string audit_directory = Path.GetDirectoryName(audit_filename); if (Directory.Exists(audit_directory)) { string audit_data = String.Format( "{0}\t{1}\t{2}\t{3}\t{4}\t{5}\r\n" , DateTime.UtcNow.ToString("yyyyMMdd.hhmmss") , ConfigurationManager.Instance.ConfigurationRecord.Account_Username , ConfigurationManager.Instance.ConfigurationRecord.Account_Nickname , Environment.UserName , filename_short , pdf_path ); try { File.AppendAllText(audit_filename, audit_data); } catch (Exception ex) { Logging.Warn(ex, "Unable to write intranet sync audit data."); } } } }
public static async Task <bool> UploadPackagedInis(IEnumerable <Archive> archives) { archives = archives.ToArray(); // defensive copy Utils.Log($"Packaging {archives.Count()} inis"); try { await using var ms = new MemoryStream(); using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true)) { foreach (var e in archives) { if (e.State == null) { continue; } var entry = z.CreateEntry(Path.GetFileName(e.Name)); await using var os = entry.Open(); await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", e.State.GetMetaIni()))); } } var client = new Common.Http.Client(); var response = await client.PostAsync($"{Consts.WabbajackBuildServerUri}indexed_files/notify", new ByteArrayContent(ms.ToArray())); if (response.IsSuccessStatusCode) { return(true); } Utils.Log("Error sending Inis"); Utils.Log(await response.Content.ReadAsStringAsync()); return(false); } catch (Exception ex) { Utils.Log(ex.ToString()); return(false); } }
private static BrowserEmulationVersion GetBrowserEmulationVersion() { BrowserEmulationVersion result; result = BrowserEmulationVersion.Default; try { RegistryKey key; key = Registry.CurrentUser.OpenSubKey(BrowserEmulationKey, true); if (key != null) { string programName; object value; programName = Path.GetFileName(Environment.GetCommandLineArgs()[0]); value = key.GetValue(programName, null); if (value != null) { result = (BrowserEmulationVersion)Convert.ToInt32(value); } } } catch (SecurityException se) { // The user does not have the permissions required to read from the registry key. logger.Error(se); } catch (UnauthorizedAccessException uae) { // The user does not have the necessary registry rights. logger.Error(uae); } return(result); }
static void AddDirToTar(Stream tar, string sourceDirectory, string app, string domain, string basePath, bool writeDirEntry = true) { // Optionally, write an entry for the directory itself. if (writeDirEntry) { TarEntry tarEntry = TarEntry.CreateEntryFromFile(sourceDirectory); tarEntry.Name = basePath; // tarOutputStream.PutNextEntry(tarEntry); ABTar.WriteTarFile(app, domain, basePath, null, tar, out var _); } // Write each file to the tar. string[] filenames = Directory.GetFiles(sourceDirectory); foreach (string filename in filenames) { using (FileStream file = File.OpenRead(filename)) { //AddFileToTarRaw(tarOutputStream, inputStream, new FileInfo(filename), // PathCombineUnixUnsafe(basePath, Path.GetFileName(filename))); ABTar.WriteTarFile(app, domain, PathCombineUnixUnsafe(basePath, Path.GetFileName(filename)), file, tar, out var _); } } // Recurse. string[] directories = Directory.GetDirectories(sourceDirectory); foreach (string directory in directories) { //AddDirToTar(tarOutputStream, directory, // PathCombineUnixUnsafe(basePath, Path.GetFileName(directory))); ABTar.WriteTarFile(app, domain, PathCombineUnixUnsafe(basePath, Path.GetFileName(directory)), null, tar, out var _); } }
public bool Initialize(string config, string game, out string errorMessage) { errorMessage = default; if (!this._client.CanInitialize(config, game)) { return(false); } this._runValidation = false; this._basePath = config; if (!Directory.Exists(this._basePath)) { errorMessage = $"{this._basePath} does not exist"; this.Log().Error(errorMessage); return(false); } try { if (!Directory.Exists(Path.Combine(this._basePath, "BannerLordLauncher Backups"))) { Directory.CreateDirectory(Path.Combine(this._basePath, "BannerLordLauncher Backups")); } } catch (Exception e) { this.Log().Error(e); } var launcherData = UserData.Load(this, Path.Combine(this._basePath, "LauncherData.xml")) ?? new UserData(); this._modulePath = Path.Combine(game, "Modules"); this.GameExeFolder = Path.Combine(game, "bin", "Win64_Shipping_Client"); var modulesFolder = Path.Combine(game, "Modules"); if (!Directory.Exists(modulesFolder)) { errorMessage = $"{modulesFolder} does not exist"; this.Log().Error(errorMessage); return(false); } var modules = Directory.EnumerateDirectories(modulesFolder, "*", SearchOption.TopDirectoryOnly) .Select(dir => Module.Load(this, Path.GetFileName(dir), game)).Where(module => module != null).ToList(); if (launcherData.SingleplayerData?.ModDatas != null) { foreach (var mod in launcherData.SingleplayerData.ModDatas) { if (this.Mods.Any(x => x.UserModData.Id.Equals(mod.Id, StringComparison.OrdinalIgnoreCase))) { continue; } var module = modules.FirstOrDefault(x => x.Id == mod.Id); if (module == null) { this.Log().Warn($"{mod.Id} could not be found in {modulesFolder}"); continue; } modules.Remove(module); var modEntry = new ModEntry { Module = module, UserModData = mod }; this.Mods.Add(modEntry); if (modEntry.Module.Official) { modEntry.IsChecked = true; } } } foreach (var module in modules) { if (this.Mods.Any(x => x.Module.Id.Equals(module.Id, StringComparison.OrdinalIgnoreCase))) { continue; } var modEntry = new ModEntry { Module = module, UserModData = new UserModData(module.Id, false) }; this.Mods.Add(modEntry); if (modEntry.Module.Official) { modEntry.IsChecked = true; } } this.AnalyzeAssemblies(); this.BuildDependencies(); this._runValidation = true; this.Validate(); return(true); }
private void BackupFiles(IList <IVssWMComponent> components, IDictionary <string, string> volumeMap, IDictionary <string, string> snapshotVolumeMap, IDictionary <string, string> vmNamesMap, Options options, ILogger logger) { var streams = new List <Stream>(); try { foreach (var component in components) { string vmBackupPath; if (options.DirectCopy) { vmBackupPath = Path.Combine(options.Output, string.Format(options.OutputFormat, vmNamesMap[component.ComponentName], component.ComponentName, DateTime.Now, "")); } else { vmBackupPath = Path.Combine(options.Output, string.Format(options.OutputFormat, vmNamesMap[component.ComponentName], component.ComponentName, DateTime.Now, options.ZipFormat ? ".zip" : ".7z")); File.Delete(vmBackupPath); } var files = new Dictionary <string, Stream>(); foreach (var file in component.Files) { string path; if (file.IsRecursive) { path = file.Path; } else { path = Path.Combine(file.Path, file.FileSpecification); } // Get the longest matching path var volumePath = volumeMap.Keys.OrderBy(o => o.Length).Reverse() .First(o => path.StartsWith(o, StringComparison.OrdinalIgnoreCase)); var volumeName = volumeMap[volumePath]; // Exclude snapshots var fileName = Path.GetFileName(path.Substring(volumePath.Length)).ToUpperInvariant(); var include = !path.EndsWith("\\*"); var pathItems = path.Split(Path.DirectorySeparatorChar); if (pathItems.Length >= 2) { if (pathItems[pathItems.Length - 2].ToLowerInvariant() == "snapshots") { include = false; } } if (include && options.VhdInclude != null) { if (options.VhdInclude.Count( x => string.CompareOrdinal(x.ToUpperInvariant(), fileName) == 0) == 0) { include = false; } } if (include && options.VhdIgnore != null) { if (options.VhdIgnore.Count( x => string.CompareOrdinal(x.ToUpperInvariant(), fileName) == 0) != 0) { include = false; } } if (include) { if (options.DirectCopy) { DoDirectCopy(vmBackupPath, snapshotVolumeMap[volumeName], volumePath.Length, path); } else { AddPathToCompressionList(files, streams, snapshotVolumeMap[volumeName], volumePath.Length, path); } } else { var errorText = $"Ignoring file {path}"; logger.Info(errorText); Console.WriteLine(errorText); } } if (!options.DirectCopy) { logger.Debug($"Start compression. File: {vmBackupPath}"); if (options.ZipFormat) { if (options.CompressionLevel == -1) { options.CompressionLevel = 6; } using (var zf = new ZipFile(vmBackupPath)) { zf.ParallelDeflateThreshold = -1; zf.UseZip64WhenSaving = Zip64Option.Always; zf.Encryption = EncryptionAlgorithm.WinZipAes256; switch (options.CompressionLevel) { case 0: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level0; break; case 1: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level1; break; case 2: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level2; break; case 3: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level3; break; case 4: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level4; break; case 5: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level5; break; case 6: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level6; break; case 7: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level7; break; case 8: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level8; break; case 9: zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level9; break; } if (BackupProgress != null) { zf.SaveProgress += (sender, e) => ReportZipProgress(component, volumeMap, e); } if (!string.IsNullOrEmpty(options.Password)) { zf.Password = options.Password; } foreach (var file in files) { logger.Debug($"Adding file: {file.Key}"); zf.AddEntry(file.Key, file.Value); } zf.Save(); } } else { if (options.CompressionLevel == -1) { options.CompressionLevel = 3; } SevenZipBase.SetLibraryPath( Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "7z.dll")); var sevenZip = new SevenZipCompressor { ArchiveFormat = OutArchiveFormat.SevenZip, CompressionMode = CompressionMode.Create, DirectoryStructure = true, PreserveDirectoryRoot = false }; if (options.MultiThreaded) { sevenZip.CustomParameters.Add("mt", "on"); } sevenZip.CustomParameters.Add("d", "24"); switch (options.CompressionLevel) { case 0: sevenZip.CompressionLevel = CompressionLevel.None; break; case 1: case 2: sevenZip.CompressionLevel = CompressionLevel.Fast; break; case 3: case 4: case 5: sevenZip.CompressionLevel = CompressionLevel.Low; break; case 6: sevenZip.CompressionLevel = CompressionLevel.Normal; break; case 7: case 8: sevenZip.CompressionLevel = CompressionLevel.High; break; case 9: sevenZip.CompressionLevel = CompressionLevel.Ultra; break; } if (BackupProgress != null) { sevenZip.FileCompressionStarted += (sender, e) => { var ebp = new BackupProgressEventArgs { AcrhiveFileName = e.FileName, Action = EventAction.StartingArchive }; _currentFile = e.FileName; Report7ZipProgress(component, volumeMap, ebp); if (_cancel) { e.Cancel = true; } }; sevenZip.FileCompressionFinished += (sender, e) => { var ebp = new BackupProgressEventArgs { AcrhiveFileName = _currentFile, Action = EventAction.ArchiveDone }; _currentFile = string.Empty; Report7ZipProgress(component, volumeMap, ebp); }; sevenZip.Compressing += (sender, e) => { var ebp = new BackupProgressEventArgs { AcrhiveFileName = _currentFile, Action = EventAction.PercentProgress, CurrentEntry = _currentFile, PercentDone = e.PercentDone }; Report7ZipProgress(component, volumeMap, ebp); }; } if (string.IsNullOrEmpty(options.Password)) { sevenZip.CompressStreamDictionary(files, vmBackupPath); } else { sevenZip.CompressStreamDictionary(files, vmBackupPath, options.Password); } } logger.Debug("Compression finished"); if (_cancel) { if (File.Exists(vmBackupPath)) { File.Delete(vmBackupPath); } throw new BackupCancelledException(); } } } } finally { // Make sure that all streams are closed foreach (var s in streams) { s.Close(); } } }
public FormattedGrepResult(GrepSearchResult result, string folderPath) { GrepResult = result; bool isFileReadOnly = Utils.IsReadOnly(GrepResult); bool isSuccess = GrepResult.IsSuccess; string basePath = string.IsNullOrWhiteSpace(folderPath) ? string.Empty : folderPath.TrimEnd('\\'); string displayedName = Path.GetFileName(GrepResult.FileNameDisplayed); if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowFilePathInResults) && GrepResult.FileNameDisplayed.Contains(basePath, StringComparison.CurrentCultureIgnoreCase)) { if (!string.IsNullOrWhiteSpace(basePath)) { displayedName = GrepResult.FileNameDisplayed.Substring(basePath.Length + 1).TrimStart('\\'); } else { displayedName = GrepResult.FileNameDisplayed; } } if (!string.IsNullOrWhiteSpace(GrepResult.AdditionalInformation)) { displayedName += " " + GrepResult.AdditionalInformation + " "; } int matchCount = (GrepResult.Matches == null ? 0 : GrepResult.Matches.Count); if (matchCount > 0) { if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowVerboseMatchCount)) { var lineCount = GrepResult.Matches.Where(r => r.LineNumber > 0) .Select(r => r.LineNumber).Distinct().Count(); displayedName = string.Format("{0} ({1} matches on {2} lines)", displayedName, matchCount, lineCount); } else { displayedName = string.Format("{0} ({1})", displayedName, matchCount); } } if (isFileReadOnly) { result.ReadOnly = true; displayedName = displayedName + " [read-only]"; } Label = displayedName; if (isFileReadOnly) { Style = "ReadOnly"; } if (!isSuccess) { Style = "Error"; } FormattedLines = new LazyResultsList(result, this); FormattedLines.LineNumberColumnWidthChanged += FormattedLines_PropertyChanged; FormattedLines.LoadFinished += FormattedLines_LoadFinished; }
public static Task <string> UploadFile(WorkQueue queue, string filename, Action <double> progressFn) { var tcs = new TaskCompletionSource <string>(); Task.Run(async() => { var client = GetAuthorizedClient(); var fsize = new FileInfo(filename).Length; var hash_task = filename.FileHashAsync(); var response = await client.PutAsync(UploadURL + $"/{Path.GetFileName(filename)}/start", new StringContent("")); if (!response.IsSuccessStatusCode) { tcs.SetException(new Exception($"Start Error: {response.StatusCode} {response.ReasonPhrase}")); return; } IEnumerable <long> Blocks(long fsize) { for (long block = 0; block *BLOCK_SIZE < fsize; block++) { yield return(block); } } var key = await response.Content.ReadAsStringAsync(); long sent = 0; using (var iqueue = new WorkQueue(MAX_CONNECTIONS)) { iqueue.Report("Starting Upload", Percent.One); await Blocks(fsize) .PMap(iqueue, async block_idx => { if (tcs.Task.IsFaulted) { return; } var block_offset = block_idx * BLOCK_SIZE; var block_size = block_offset + BLOCK_SIZE > fsize ? fsize - block_offset : BLOCK_SIZE; Interlocked.Add(ref sent, block_size); progressFn((double)sent / fsize); int retries = 0; using (var fs = File.OpenRead(filename)) { fs.Position = block_offset; var data = new byte[block_size]; await fs.ReadAsync(data, 0, data.Length); response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}", new ByteArrayContent(data)); if (!response.IsSuccessStatusCode) { tcs.SetException(new Exception($"Put Error: {response.StatusCode} {response.ReasonPhrase}")); return; } var val = long.Parse(await response.Content.ReadAsStringAsync()); if (val != block_offset + data.Length) { tcs.SetResult($"Sync Error {val} vs {block_offset + data.Length}"); tcs.SetException(new Exception($"Sync Error {val} vs {block_offset + data.Length}")); } } }); } if (!tcs.Task.IsFaulted) { progressFn(1.0); var hash = (await hash_task).FromBase64().ToHex(); response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent("")); if (response.IsSuccessStatusCode) { tcs.SetResult(await response.Content.ReadAsStringAsync()); } else { tcs.SetException(new Exception($"Finalization Error: {response.StatusCode} {response.ReasonPhrase}")); } } progressFn(0.0); }); return(tcs.Task); }
// ************************************************************************************************************* // *** MIGRATION TO OPEN SOURCE CODE *************************************************************************** // ************************************************************************************************************* private void AddLegacyWebLibrariesThatCanBeFoundOnDisk() { /** * Plan: * Iterate through all the folders in the Qiqqa data directory * If a folder contains a valid Library record and it is a WEB library, then add it to our list with the word '[LEGACY]' in front of it */ string base_directory_path = UpgradePaths.V037To038.SQLiteUpgrade.BaseDirectoryForQiqqa; Logging.Info("Going to scan for web libraries at: {0}", base_directory_path); if (Directory.Exists(base_directory_path)) { string[] library_directories = Directory.GetDirectories(base_directory_path); foreach (string library_directory in library_directories) { Logging.Info("Inspecting directory {0} - Phase 1 : Web & Known Libraries", library_directory); string databaselist_file = Path.GetFullPath(Path.Combine(library_directory, @"Qiqqa.known_web_libraries")); if (File.Exists(databaselist_file)) { LoadKnownWebLibraries(databaselist_file, true); } } foreach (string library_directory in library_directories) { Logging.Info("Inspecting directory {0} - Phase 2 : Intranet Libraries", library_directory); string databaselist_file = IntranetLibraryTools.GetLibraryDetailPath(library_directory); if (File.Exists(databaselist_file)) { IntranetLibraryDetail intranet_library_detail = IntranetLibraryDetail.Read(databaselist_file); UpdateKnownWebLibraryFromIntranet(library_directory, extra_info_message_on_skip: String.Format(" as obtained from file {0}", databaselist_file)); } } foreach (string library_directory in library_directories) { Logging.Info("Inspecting directory {0} - Phase 3 : Bundles", library_directory); // must be a qiqqa_bundle and/or qiqqa_bundle_manifest file set Logging.Warn("Auto bundle import at startup is not yet suppoerted."); } foreach (string library_directory in library_directories) { Logging.Info("Inspecting directory {0} - Phase 4 : Local and Legacy Libraries", library_directory); string database_file = Path.GetFullPath(Path.Combine(library_directory, @"Qiqqa.library")); if (File.Exists(database_file)) { var library_id = Path.GetFileName(library_directory); WebLibraryDetail new_web_library_detail = new WebLibraryDetail(); new_web_library_detail.Id = library_id; new_web_library_detail.Title = "Legacy Web Library - " + new_web_library_detail.Id; new_web_library_detail.IsReadOnly = false; // library: UNKNOWN type UpdateKnownWebLibrary(new_web_library_detail); } } } }
public AppCompatCache(string filename, int controlSet, bool noLogs) { byte[] rawBytes = null; Caches = new List <IAppCompatCache>(); var controlSetIds = new List <int>(); RegistryKey subKey = null; var isLiveRegistry = string.IsNullOrEmpty(filename); if (isLiveRegistry) { var keyCurrUser = Microsoft.Win32.Registry.LocalMachine; var subKey2 = keyCurrUser.OpenSubKey(@"SYSTEM\CurrentControlSet\Control\Session Manager\AppCompatCache"); if (subKey2 == null) { subKey2 = keyCurrUser.OpenSubKey(@"SYSTEM\CurrentControlSet\Control\Session Manager\AppCompatibility"); if (subKey2 == null) { Console.WriteLine( @"'CurrentControlSet\Control\Session Manager\AppCompatCache' key not found! Exiting"); return; } } rawBytes = (byte[])subKey2.GetValue("AppCompatCache", null); subKey2 = keyCurrUser.OpenSubKey(@"SYSTEM\Select"); ControlSet = (int)subKey2.GetValue("Current"); var is32Bit = Is32Bit(filename, null); var cache = Init(rawBytes, is32Bit, ControlSet); Caches.Add(cache); return; } RegistryHive reg; Privilege[] privileges = { Privilege.EnableDelegation, Privilege.Impersonate, Privilege.Tcb }; using (new PrivilegeEnabler(Privilege.Backup, privileges)) { ControlSet = controlSet; if (File.Exists(filename) == false && RawCopy.Helper.RawFileExists(filename) == false) { throw new FileNotFoundException($"File not found ({filename})!"); } var dirname = Path.GetDirectoryName(filename); var hiveBase = Path.GetFileName(filename); List <RawCopy.RawCopyReturn> rawFiles = null; try { reg = new RegistryHive(filename) { RecoverDeleted = true }; } catch (IOException) { //file is in use if (RawCopy.Helper.IsAdministrator() == false) { throw new UnauthorizedAccessException("Administrator privileges not found!"); } _logger.Warn($"'{filename}' is in use. Rerouting...\r\n"); var files = new List <string>(); files.Add(filename); var logFiles = Directory.GetFiles(dirname, $"{hiveBase}.LOG?").ToList(); var log1 = $"{dirname}\\{hiveBase}.LOG1"; var log2 = $"{dirname}\\{hiveBase}.LOG2"; if (logFiles.Count == 0) { if (RawCopy.Helper.RawFileExists(log1)) { logFiles.Add(log1); } if (RawCopy.Helper.RawFileExists(log2)) { logFiles.Add(log2); } } foreach (var logFile in logFiles) { files.Add(logFile); } rawFiles = RawCopy.Helper.GetFiles(files); reg = new RegistryHive(rawFiles.First().FileBytes, rawFiles.First().InputFilename); } if (reg.Header.PrimarySequenceNumber != reg.Header.SecondarySequenceNumber) { if (string.IsNullOrEmpty(dirname)) { dirname = "."; } var logFiles = Directory.GetFiles(dirname, $"{hiveBase}.LOG?").ToList(); var log1 = $"{dirname}\\{hiveBase}.LOG1"; var log2 = $"{dirname}\\{hiveBase}.LOG2"; if (logFiles.Count == 0) { if (File.Exists(log1)) { logFiles.Add(log1); } if (File.Exists(log2)) { logFiles.Add(log2); } } if (logFiles.Count == 0) { if (RawCopy.Helper.RawFileExists(log1)) { logFiles.Add(log1); } if (RawCopy.Helper.RawFileExists(log2)) { logFiles.Add(log2); } } if (logFiles.Count == 0) { if (noLogs == false) { _logger.Warn("Registry hive is dirty and no transaction logs were found in the same directory! LOGs should have same base name as the hive. Aborting!!"); throw new Exception("Sequence numbers do not match and transaction logs were not found in the same directory as the hive. Aborting"); } else { _logger.Warn("Registry hive is dirty and no transaction logs were found in the same directory. Data may be missing! Continuing anyways..."); } } else { if (noLogs == false) { if (rawFiles != null) { var lt = new List <TransactionLogFileInfo>(); foreach (var rawCopyReturn in rawFiles.Skip(1).ToList()) { var tt = new TransactionLogFileInfo(rawCopyReturn.InputFilename, rawCopyReturn.FileBytes); lt.Add(tt); } reg.ProcessTransactionLogs(lt, true); } else { reg.ProcessTransactionLogs(logFiles.ToList(), true); } } else { _logger.Warn("Registry hive is dirty and transaction logs were found in the same directory, but --nl was provided. Data may be missing! Continuing anyways..."); } } } reg.ParseHive(); } if (controlSet == -1) { for (var i = 0; i < 10; i++) { subKey = reg.GetKey($@"ControlSet00{i}\Control\Session Manager\AppCompatCache"); if (subKey == null) { subKey = reg.GetKey($@"ControlSet00{i}\Control\Session Manager\AppCompatibility"); } if (subKey != null) { controlSetIds.Add(i); } } if (controlSetIds.Count > 1) { _logger.Warn( $"***The following ControlSet00x keys will be exported: {string.Join(",", controlSetIds)}. Use -c to process keys individually\r\n"); } } else { //a control set was passed in subKey = reg.GetKey($@"ControlSet00{ControlSet}\Control\Session Manager\AppCompatCache"); if (subKey == null) { subKey = reg.GetKey($@"ControlSet00{ControlSet}\Control\Session Manager\AppCompatibility"); } if (subKey == null) { throw new Exception($"Could not find ControlSet00{ControlSet}. Exiting"); } controlSetIds.Add(ControlSet); } var is32 = Is32Bit(filename, reg); _logger.Debug($@"**** Found {controlSetIds.Count} ids to process"); foreach (var id in controlSetIds) { _logger.Debug($@"**** Processing id {id}"); // var hive2 = new RegistryHiveOnDemand(filename); subKey = reg.GetKey($@"ControlSet00{id}\Control\Session Manager\AppCompatCache"); if (subKey == null) { _logger.Debug($@"**** Initial subkey null, getting appCompatability key"); subKey = reg.GetKey($@"ControlSet00{id}\Control\Session Manager\AppCompatibility"); } _logger.Debug($@"**** Looking AppCompatcache value"); var val = subKey?.Values.SingleOrDefault(c => c.ValueName == "AppCompatCache"); if (val != null) { _logger.Debug($@"**** Found AppCompatcache value"); rawBytes = val.ValueDataRaw; } if (rawBytes == null) { _logger.Error($@"'AppCompatCache' value not found for 'ControlSet00{id}'! Exiting"); } var cache = Init(rawBytes, is32, id); Caches.Add(cache); } }
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path, string rel_path, bool topLevel) { var hash = abs_path.FileHash(); var fi = new FileInfo(abs_path); if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(abs_path)) { var result = await TryGetContentsFromServer(hash); if (result != null) { Utils.Log($"Downloaded VFS data for {Path.GetFileName(abs_path)}"); VirtualFile Convert(IndexedVirtualFile file, string path, VirtualFile vparent) { var vself = new VirtualFile { Context = context, Name = path, Parent = vparent, Size = file.Size, LastModified = fi.LastWriteTimeUtc.Ticks, LastAnalyzed = DateTime.Now.Ticks, Hash = file.Hash, }; vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList(); return(vself); } return(Convert(result, rel_path, parent)); } } var self = new VirtualFile { Context = context, Name = rel_path, Parent = parent, Size = fi.Length, LastModified = fi.LastWriteTimeUtc.Ticks, LastAnalyzed = DateTime.Now.Ticks, Hash = hash }; if (context.UseExtendedHashes) { self.ExtendedHashes = ExtendedHashes.FromFile(abs_path); } if (FileExtractor.CanExtract(abs_path)) { using (var tempFolder = context.GetTemporaryFolder()) { await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName); var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories) .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName), false)); self.Children = list.ToImmutableList(); } } return(self); }
public FormattedGrepResult(GrepSearchResult result, string folderPath) { grepResult = result; fileInfo = new FileInfo(grepResult.FileNameReal); bool isFileReadOnly = Utils.IsReadOnly(grepResult); bool isSuccess = grepResult.IsSuccess; string basePath = string.IsNullOrWhiteSpace(folderPath) ? string.Empty : Utils.GetBaseFolder(folderPath).TrimEnd('\\'); string displayedName = Path.GetFileName(grepResult.FileNameDisplayed); if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowFilePathInResults) && grepResult.FileNameDisplayed.Contains(basePath)) { if (!string.IsNullOrWhiteSpace(basePath)) { displayedName = grepResult.FileNameDisplayed.Substring(basePath.Length + 1).TrimStart('\\'); } else { displayedName = grepResult.FileNameDisplayed; } } if (!string.IsNullOrWhiteSpace(grepResult.AdditionalInformation)) { displayedName += " " + grepResult.AdditionalInformation + " "; } int matchCount = (grepResult.Matches == null ? 0 : grepResult.Matches.Count); if (matchCount > 0) { if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowVerboseMatchCount)) { var lineCount = grepResult.Matches.Where(r => r.LineNumber > 0) .Select(r => r.LineNumber).Distinct().Count(); displayedName = string.Format("{0} ({1} matches on {2} lines)", displayedName, matchCount, lineCount); } else { displayedName = string.Format("{0} ({1})", displayedName, matchCount); } } if (isFileReadOnly) { result.ReadOnly = true; displayedName = displayedName + " [read-only]"; } label = displayedName; if (isFileReadOnly) { style = "ReadOnly"; } if (!isSuccess) { style = "Error"; } formattedLines = new LazyResultsList(result, this); formattedLines.LineNumberColumnWidthChanged += formattedLines_PropertyChanged; formattedLines.LoadFinished += formattedLines_LoadFinished; if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ExpandResults)) { IsExpanded = true; } }
// ************************************************************************************************************* // *** MIGRATION TO OPEN SOURCE CODE *************************************************************************** // ************************************************************************************************************* private void AddLegacyWebLibrariesThatCanBeFoundOnDisk() { WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread(); try { ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(AddLegacyWebLibrariesThatCanBeFoundOnDisk)); /** * Plan: * - Iterate through all the folders in the Qiqqa data directory. * - If a folder contains a valid Library record and it is a WEB library, * then add it to our list with the word '[LEGACY]' in front of it. */ string base_directory_path = UpgradePaths.V037To038.SQLiteUpgrade.BaseDirectoryForQiqqa; Logging.Info("Going to scan for web libraries at: {0}", base_directory_path); if (Directory.Exists(base_directory_path)) { string[] library_directories = Directory.GetDirectories(base_directory_path); foreach (string library_directory in library_directories) { Logging.Info("Inspecting directory {0} - Phase 1 : Web & Known Libraries", library_directory); string databaselist_file = Path.GetFullPath(Path.Combine(library_directory, @"Qiqqa.known_web_libraries")); if (File.Exists(databaselist_file)) { LoadKnownWebLibraries(databaselist_file, only_load_those_libraries_which_are_actually_present: true); } } foreach (string library_directory in library_directories) { Logging.Info("Inspecting directory {0} - Phase 2 : Intranet Libraries", library_directory); string databaselist_file = IntranetLibraryTools.GetLibraryDetailPath(library_directory); if (File.Exists(databaselist_file)) { IntranetLibraryDetail intranet_library_detail = IntranetLibraryDetail.Read(databaselist_file); UpdateKnownWebLibraryFromIntranet(library_directory, extra_info_message_on_skip: String.Format(" as obtained from file {0}", databaselist_file)); } } foreach (string library_directory in library_directories) { Logging.Info("Inspecting directory {0} - Phase 3 : Bundles", library_directory); // must be a qiqqa_bundle and/or qiqqa_bundle_manifest file set Logging.Warn("Auto bundle import at startup is not yet supported."); } foreach (string library_directory in library_directories) { Logging.Info("Inspecting directory {0} - Phase 4 : Local and Legacy Libraries", library_directory); string database_file = LibraryDB.GetLibraryDBPath(library_directory); string db_syncref_path = IntranetLibraryTools.GetLibraryMetadataPath(library_directory); // add/update only if this is not a Internet sync directory/DB! if (File.Exists(db_syncref_path)) { Logging.Info("Skip the Qiqqa Internet/Intranet Sync directory and the sync DB contained therein: '{0}'", db_syncref_path); // https://github.com/jimmejardine/qiqqa-open-source/issues/145 :: delete lib file when it is very small and was illegally // constructed by a previous v82beta Qiqqa release: if (File.Exists(database_file)) { long s3length = File.GetSize(database_file); if (6 * 1024 > s3length) { Logging.Warn("DELETE the wrongfully created DB file '{0}' in the Qiqqa Internet/Intranet Sync directory and the sync DB contained therein: '{1}', which has precedence!", database_file, db_syncref_path); FileTools.DeleteToRecycleBin(database_file); } else { Logging.Error("Inspect the Library DB file '{0}' in the Qiqqa Internet/Intranet Sync directory and the sync DB contained therein: '{1}', which MAY have precedence. Delete one of these manually to clean up your system as Qiqqa heuristics cannot tell which is the prevalent metadata database here!", database_file, db_syncref_path); } } continue; } if (File.Exists(database_file)) { var library_id = Path.GetFileName(library_directory); WebLibraryDetail new_web_library_detail = new WebLibraryDetail(); new_web_library_detail.Id = library_id; new_web_library_detail.Title = "Legacy Web Library - " + new_web_library_detail.Id; new_web_library_detail.IsReadOnly = false; // library: UNKNOWN type UpdateKnownWebLibrary(new_web_library_detail); } } } } catch (Exception ex) { Logging.Error(ex, "There was a problem while scanning for (legacy) libraries."); } }
private void BackupFiles(IList <IVssWMComponent> components, IDictionary <string, string> volumeMap, IDictionary <string, string> snapshotVolumeMap, IDictionary <string, string> vmNamesMap, Options options) { IList <System.IO.Stream> streams = new List <System.IO.Stream>(); try { foreach (var component in components) { string vmBackupPath; if (options.DirectCopy) { vmBackupPath = Path.Combine(options.Output, string.Format(options.OutputFormat, vmNamesMap[component.ComponentName], component.ComponentName, DateTime.Now, "")); } else { vmBackupPath = Path.Combine(options.Output, string.Format(options.OutputFormat, vmNamesMap[component.ComponentName], component.ComponentName, DateTime.Now, options.ZipFormat ? ".zip" : ".7z")); File.Delete(vmBackupPath); } var files = new Dictionary <string, System.IO.Stream>(); foreach (var file in component.Files) { string path; if (file.IsRecursive) { path = file.Path; } else { path = Path.Combine(file.Path, file.FileSpecification); } // Get the longest matching path var volumePath = volumeMap.Keys.OrderBy(o => o.Length).Reverse().First(o => path.StartsWith(o, StringComparison.OrdinalIgnoreCase)); var volumeName = volumeMap[volumePath]; // Exclude snapshots var fileName = Path.GetFileName(path.Substring(volumePath.Length)).ToUpperInvariant(); var include = !path.EndsWith("\\*"); var pathItems = path.Split(Path.DirectorySeparatorChar); if (pathItems.Length >= 2) { if (pathItems[pathItems.Length - 2].ToLowerInvariant() == "snapshots") { include = false; } } if (include && options.VhdInclude != null) { if (options.VhdInclude.Count(x => string.CompareOrdinal(x.ToUpperInvariant(), fileName) == 0) == 0) { include = false; } } if (include && options.VhdIgnore != null) { if (options.VhdIgnore.Count(x => string.CompareOrdinal(x.ToUpperInvariant(), fileName) == 0) != 0) { include = false; } } if (include) { if (options.DirectCopy) { DoDirectCopy(vmBackupPath, snapshotVolumeMap[volumeName], volumePath.Length, path); } else { AddPathToSevenZip(files, streams, snapshotVolumeMap[volumeName], volumePath.Length, path); } } else { Console.WriteLine("Ignoring file {0}", path); } } if (!options.DirectCopy) { SevenZipBase.SetLibraryPath(Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "7z.dll")); var sevenZip = new SevenZipCompressor { ArchiveFormat = options.ZipFormat ? OutArchiveFormat.Zip : OutArchiveFormat.SevenZip, CompressionMode = CompressionMode.Create, DirectoryStructure = true, PreserveDirectoryRoot = false }; sevenZip.CustomParameters.Add("mt", "on"); switch (options.CompressionLevel) { case 0: sevenZip.CompressionLevel = CompressionLevel.None; break; case 1: sevenZip.CompressionLevel = CompressionLevel.Fast; break; case 2: sevenZip.CompressionLevel = CompressionLevel.Fast; break; case 3: sevenZip.CompressionLevel = CompressionLevel.Low; break; case 4: sevenZip.CompressionLevel = CompressionLevel.Low; break; case 5: sevenZip.CompressionLevel = CompressionLevel.Low; break; case 6: sevenZip.CompressionLevel = CompressionLevel.Normal; break; case 7: sevenZip.CompressionLevel = CompressionLevel.High; break; case 8: sevenZip.CompressionLevel = CompressionLevel.High; break; case 9: sevenZip.CompressionLevel = CompressionLevel.Ultra; break; } if (BackupProgress != null) { sevenZip.FileCompressionStarted += (sender, e) => { var ebp = new BackupProgressEventArgs() { AcrhiveFileName = e.FileName, Action = EventAction.StartingArchive }; _sevenZipCurrentFile = e.FileName; Report7ZipProgress(component, volumeMap, ebp); if (_cancel) { e.Cancel = true; } }; sevenZip.FileCompressionFinished += (sender, e) => { var ebp = new BackupProgressEventArgs { AcrhiveFileName = _sevenZipCurrentFile, Action = EventAction.ArchiveDone }; _sevenZipCurrentFile = string.Empty; Report7ZipProgress(component, volumeMap, ebp); }; sevenZip.Compressing += (sender, e) => { var ebp = new BackupProgressEventArgs { AcrhiveFileName = _sevenZipCurrentFile, Action = EventAction.PercentProgress, CurrentEntry = _sevenZipCurrentFile, PercentDone = e.PercentDone }; Report7ZipProgress(component, volumeMap, ebp); if (_cancel) { e.Cancel = true; } }; } if (string.IsNullOrEmpty(options.Password)) { sevenZip.CompressStreamDictionary(files, vmBackupPath); } else { sevenZip.CompressStreamDictionary(files, vmBackupPath, options.Password); } if (_cancel) { if (File.Exists(vmBackupPath)) { File.Delete(vmBackupPath); } throw new BackupCancelledException(); } } } } finally { // Make sure that all streams are closed foreach (var s in streams) { s.Close(); } } }
private string MkLegalSizedPath(string basename, string typeIdStr) { const int PATH_MAX = 240; // must be less than 255 / 260 - see also https://kb.acronis.com/content/39790 string root = Path.GetDirectoryName(basename); string name = Path.GetFileName(basename); string dataname = Path.GetFileNameWithoutExtension(DataFile); string ext = SubStr(Path.GetExtension(DataFile), 1).Trim(); // produce the extension without leading dot if (ext.StartsWith("bib")) { ext = SubStr(ext, 3).Trim(); } if (ext.Length > 0) { ext = "." + ext; } // UNC long filename/path support by forcing this to be a UNC path: string filenamebase = $"{dataname}.{name}{ext}{ExtensionWithDot}"; // first make the full path without the approved/received, so that that bit doesn't make a difference // in the length check and subsequent decision to produce a shorthand filename path or not: // It's not always needed, but do the different shorthand conversions anyway and pick the longest fitting one: string short_tn = SanitizeFilename(CamelCaseShorthand(name)); string short_dn = SanitizeFilename(SubStr(dataname, 0, 10) + CamelCaseShorthand(dataname)); string hash = StreamMD5.FromText(filenamebase).ToUpper(); string short_hash = SubStr(hash, 0, Math.Max(6, 11 - short_tn.Length)); // this variant will fit in the length criterium, guaranteed: string alt_filepath0 = Path.GetFullPath(Path.Combine(root, $"{short_dn}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}")); string filepath = alt_filepath0; // next, we construct the longer variants to check if they fit. // // DO NOTE that we create a path without typeIdStr part first, because we want both received and approved files to be based // on the *same* alt selection decision! string picked_alt_filepath = Path.GetFullPath(Path.Combine(root, $"{short_dn}.{short_hash}_{short_tn}{ext}.APPROVEDXYZ{ExtensionWithDot}")); name = SanitizeFilename(name); dataname = SanitizeFilename(dataname); string alt_filepath1 = Path.GetFullPath(Path.Combine(root, $"{short_dn}_{short_hash}.{name}{ext}.APPROVEDXYZ{ExtensionWithDot}")); if (alt_filepath1.Length < PATH_MAX) { filepath = Path.GetFullPath(Path.Combine(root, $"{short_dn}_{short_hash}.{name}{ext}{typeIdStr}{ExtensionWithDot}")); picked_alt_filepath = alt_filepath1; } // second alternative: only pick this one if it fits and produces a longer name: string alt_filepath2 = Path.GetFullPath(Path.Combine(root, $"{dataname}.{short_hash}_{short_tn}{ext}.APPROVEDXYZ{ExtensionWithDot}")); if (alt_filepath2.Length < PATH_MAX && alt_filepath2.Length > picked_alt_filepath.Length) { filepath = Path.GetFullPath(Path.Combine(root, $"{dataname}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}")); picked_alt_filepath = alt_filepath2; } else { // third alt: the 'optimally trimmed' test name used as part of the filename: int trim_length = PATH_MAX - alt_filepath0.Length + 10 - 1; string short_dn2 = SanitizeFilename(SubStr(dataname, 0, trim_length) + CamelCaseShorthand(dataname)); string alt_filepath3 = Path.GetFullPath(Path.Combine(root, $"{short_dn2}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}")); if (alt_filepath3.Length < PATH_MAX && alt_filepath3.Length > picked_alt_filepath.Length) { filepath = Path.GetFullPath(Path.Combine(root, $"{short_dn2}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}")); picked_alt_filepath = alt_filepath3; } } // fourth alt: the full, unadulterated path; if it fits in the length criterium, take it anyway string alt_filepath4 = Path.GetFullPath(Path.Combine(root, $"{dataname}.{name}{ext}.APPROVEDXYZ{ExtensionWithDot}")); if (alt_filepath4.Length < PATH_MAX) { // UNC long filename/path support by forcing this to be a UNC path: filepath = Path.GetFullPath(Path.Combine(root, $"{dataname}.{name}{ext}{typeIdStr}{ExtensionWithDot}")); picked_alt_filepath = alt_filepath4; } return(filepath); }
public override async Task <JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) { using (var queue = new WorkQueue(4)) { Utils.Log($"Indexing game files"); var states = GameRegistry.Games.Values .Where(game => game.GameLocation() != null && game.MainExecutable != null) .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories) .Select(file => new GameFileSourceDownloader.State { Game = game.Game, GameVersion = game.InstalledVersion, GameFile = file.RelativeTo(game.GameLocation()), })) .ToList(); var pks = states.Select(s => s.PrimaryKeyString).Distinct().ToArray(); Utils.Log($"Found {pks.Length} archives to cross-reference with the database"); var found = (await db.DownloadStates .AsQueryable().Where(s => pks.Contains(s.Key)) .Select(s => s.Key) .ToListAsync()) .ToDictionary(s => s); states = states.Where(s => !found.ContainsKey(s.PrimaryKeyString)).ToList(); Utils.Log($"Found {states.Count} archives to index"); await states.PMap(queue, async state => { var path = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile); Utils.Log($"Hashing Game file {path}"); try { state.Hash = await path.FileHashAsync(); } catch (IOException) { Utils.Log($"Unable to hash {path}"); } }); var with_hash = states.Where(state => state.Hash != null).ToList(); Utils.Log($"Inserting {with_hash.Count} jobs."); var jobs = states.Select(state => new IndexJob { Archive = new Archive { Name = Path.GetFileName(state.GameFile), State = state } }) .Select(j => new Job { Payload = j, RequiresNexus = j.UsesNexus }) .ToList(); if (jobs.Count > 0) { await db.Jobs.InsertManyAsync(jobs); } return(JobResult.Success()); } }