public static async Task UploadPackagedInis(IEnumerable <IndexedArchive> archives)
{
archives = archives.ToArray(); // defensive copy
Utils.Log($"Packaging {archives.Count()} inis");
try
{
await using var ms = new MemoryStream();
using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true))
{
foreach (var archive in archives)
{
var state = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(archive.IniData));
var entry = z.CreateEntry(Path.GetFileName(archive.Name));
await using var os = entry.Open();
await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", state.GetMetaIni())));
}
}
var webClient = new WebClient();
await webClient.UploadDataTaskAsync($"https://{Consts.WabbajackCacheHostname}/indexed_files/notify",
"POST", ms.ToArray());
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
}
public override async ValueTask <Directive> Run(RawSourceFile source)
{
var filename = Path.GetFileName(source.Path);
var gameFile = Path.Combine(_mo2Compiler.GamePath, "Data", filename);
if (!Consts.GameESMs.Contains(filename) || !source.Path.StartsWith("mods\\") ||
!File.Exists(gameFile))
{
return(null);
}
Utils.Log(
$"An ESM named {filename} was found in a mod that shares a name with one of the core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched");
var result = source.EvolveTo <CleanedESM>();
result.SourceESMHash = _compiler.VFS.Index.ByRootPath[gameFile].Hash;
Utils.Status($"Generating patch of {filename}");
await using (var ms = new MemoryStream())
{
await Utils.CreatePatch(File.ReadAllBytes(gameFile), File.ReadAllBytes(source.AbsolutePath), ms);
var data = ms.ToArray();
result.SourceDataID = _compiler.IncludeFile(data);
Utils.Log($"Generated a {data.Length} byte patch for {filename}");
}
return(result);
}
public object Convert(object value, Type targetType, object parameter, CultureInfo culture)
{
string _path = value as string;
if (File.Exists(_path))
{
if (_path.Length > 240) // 规避文件名和路径名过长问题
{
if (!Directory.Exists("./temp"))
{
Directory.CreateDirectory("./temp");
}
FileInfo fi = new FileInfo(_path);
string _tmp = Path.GetFileName(_path);
if (!File.Exists("./temp/" + _tmp))
{
fi.CopyTo("./temp/" + _tmp, true);
}
return(new BitmapImage(new Uri(AppDomain.CurrentDomain.BaseDirectory + @"./temp/" + _tmp, UriKind.RelativeOrAbsolute)));
}
else
{
return(new BitmapImage(new Uri(_path, UriKind.RelativeOrAbsolute)));
}
}
else
{
return(null);
}
}
internal static void QueueUploadOfMissingPDFs(Library library, List <PDFDocument> pdf_documents)
{
// Get a list of all the documents in the Intranet library
List <string> existing_pdfs_full = IntranetLibraryTools.GetListOfDocumentsInLibrary(library.WebLibraryDetail.IntranetPath);
HashSet <string> existing_pdfs = new HashSet <string>();
foreach (string existing_pdf in existing_pdfs_full)
{
existing_pdfs.Add(Path.GetFileName(existing_pdf));
}
foreach (PDFDocument pdf_document in pdf_documents)
{
bool deleted = pdf_document.Deleted;
// Try to upload all files that we have
if (!deleted && pdf_document.DocumentExists)
{
string filename_full = PDFDocumentFileLocations.DocumentPath(library, pdf_document.Fingerprint, "pdf");
string filename_short = Path.GetFileName(filename_full);
if (!existing_pdfs.Contains(filename_short))
{
SyncQueues.Instance.QueuePut(pdf_document.Fingerprint, library);
}
}
}
}
private static string ConvertNormalFilenameToZoteroFilename(string bibtex_base_filename, string filename)
{
string bibtex_base_path = Path.GetDirectoryName(bibtex_base_filename);
string file = Path.GetFileName(filename);
string path = filename;
/// Try shrink it to a relative path
if (path.StartsWith(bibtex_base_path))
{
path = path.Substring(bibtex_base_path.Length + 1);
}
else
{
Logging.Warn("Unable to convert path to Zotero form: {0}", path);
}
// Replace the slashes
path = path.Replace('\\', '/');
// Concatenate
string export_string =
""
+ file
+ ":"
+ path
+ ":"
+ "application/pdf";
return(export_string);
}
private void ChooseCSLFile(CSLFileSource csl_file_source)
{
OpenFileDialog dialog = new OpenFileDialog();
dialog.Filter = "CSL style files|*.csl" + "|" + "All files|*.*";
dialog.CheckFileExists = true;
dialog.Multiselect = false;
string filename = null;
string directory = null;
// Decide what we want to use as the CSL file location
if (CSLFileSource.TEXTBOX == csl_file_source)
{
try
{
CorrectStyleFilenameForNewDirectoryLocation();
directory = Path.GetDirectoryName(ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile);
filename = Path.GetFileName(ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile);
// If the directory no longer exists, kill our memory of it. We will pick the default directory again.
if (!Directory.Exists(directory))
{
directory = null;
}
}
catch { }
}
if (null == directory)
{
directory = PDFDocumentCitingTools.BASE_STYLE_DIRECTORY;
}
// Set the dialog defaults
dialog.FileName = filename;
dialog.InitialDirectory = directory;
// Get the user response
if (true == dialog.ShowDialog())
{
ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile = dialog.FileName;
ConfigurationManager.Instance.ConfigurationRecord_Bindable.NotifyPropertyChanged(() => ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile);
// Check if it is a dependent style
string style_xml_filename = ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile;
string root_style_filename = DependentStyleDetector.GetRootStyleFilename(style_xml_filename);
if (root_style_filename != style_xml_filename)
{
if (null != root_style_filename)
{
MessageBoxes.Info("This style is dependent on another style, so InCite will be using this file instead:\n" + root_style_filename);
}
else
{
MessageBoxes.Info("This style is dependent on another style that is not available on your computer. Please download it before proceeding.");
}
}
}
}
/// <summary>
/// MurMur3 hashes the file pointed to by this string
/// </summary>
/// <param name="file"></param>
/// <returns></returns>
public static string FileSHA256(this string file)
{
var sha = new SHA256Managed();
using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write))
{
using (var i = File.OpenRead(file))
i.CopyToWithStatus(new FileInfo(file).Length, o, $"Hashing {Path.GetFileName(file)}");
}
return(sha.Hash.ToBase64());
}
public static StreamReader openStreamReaderWithLocalCheck(String filename)
{
if (File.Exists(filename))
{
return(new StreamReader(filename));
}
else
{
return(new StreamReader(Path.GetFileName(filename)));
}
}
internal static void DaemonPut(Library library, string fingerprint)
{
string filename_full = PDFDocumentFileLocations.DocumentPath(library, fingerprint, "pdf");
string filename_short = Path.GetFileName(filename_full);
string pdf_path = IntranetLibraryTools.GetLibraryPDFPath(library.WebLibraryDetail.IntranetPath, filename_short);
DirectoryTools.CreateDirectory(Path.GetDirectoryName(pdf_path));
Logging.Info("+Copying up {0}", fingerprint);
File.Copy(filename_full, pdf_path);
Logging.Info("-Copying up {0}", fingerprint);
}
private void UpdateArchive(VirtualFile f)
{
if (!f.IsStaged)
{
throw new InvalidDataException("Can't analyze an unstaged file");
}
var tmp_dir = Path.Combine(_stagedRoot, Guid.NewGuid().ToString());
Utils.Status($"Extracting Archive {Path.GetFileName(f.StagedPath)}");
FileExtractor.ExtractAll(f.StagedPath, tmp_dir);
Utils.Status($"Updating Archive {Path.GetFileName(f.StagedPath)}");
var entries = Directory.EnumerateFiles(tmp_dir, "*", SearchOption.AllDirectories)
.Select(path => path.RelativeTo(tmp_dir));
var new_files = entries.Select(e =>
{
var new_path = new string[f.Paths.Length + 1];
f.Paths.CopyTo(new_path, 0);
new_path[f.Paths.Length] = e;
var nf = new VirtualFile
{
Paths = new_path
};
nf._stagedPath = Path.Combine(tmp_dir, e);
Add(nf);
return(nf);
}).ToList();
// Analyze them
new_files.PMap(file =>
{
Utils.Status($"Analyzing {Path.GetFileName(file.StagedPath)}");
file.Analyze();
});
// Recurse into any archives in this archive
new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(file));
f.FinishedIndexing = true;
if (!_isSyncing)
{
SyncToDisk();
}
Utils.Status("Cleaning Directory");
DeleteDirectory(tmp_dir);
}
private void BackupFile(string file)
{
var path = Path.Combine(this._basePath, "BannerLordLauncher Backups");
try
{
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
}
catch (Exception e)
{
this.Log().Error(e);
return;
}
if (!File.Exists(file))
{
return;
}
var ext = Path.GetExtension(file);
var i = 0;
var newFile = Path.ChangeExtension(file, $"{ext}.{i:D3}");
Debug.Assert(newFile != null, nameof(newFile) + " != null");
newFile = Path.Combine(path, Path.GetFileName(newFile));
while (File.Exists(newFile))
{
i++;
newFile = Path.ChangeExtension(file, $"{ext}.{i:D3}");
Debug.Assert(newFile != null, nameof(newFile) + " != null");
newFile = Path.Combine(path, Path.GetFileName(newFile));
}
if (i > 999)
{
return;
}
try
{
Debug.Assert(file != null, nameof(file) + " != null");
File.Move(file, newFile);
}
catch (Exception e)
{
this.Log().Error(e);
}
}
private static bool SetBrowserEmulationVersion(BrowserEmulationVersion browserEmulationVersion)
{
bool result;
result = false;
try
{
RegistryKey key;
key = Registry.CurrentUser.OpenSubKey(BrowserEmulationKey, true);
if (key != null)
{
string programName;
programName = Path.GetFileName(Environment.GetCommandLineArgs()[0]);
if (browserEmulationVersion != BrowserEmulationVersion.Default)
{
// if it's a valid value, update or create the value
key.SetValue(programName, (int)browserEmulationVersion, RegistryValueKind.DWord);
logger.Warn("SETTING REGISTRY:{0}-{1}-{2}-{3}", key.Name, programName, (int)browserEmulationVersion, RegistryValueKind.DWord.ToString());
}
else
{
// otherwise, remove the existing value
key.DeleteValue(programName, false);
logger.Warn("DELETING REGISTRY KEY:{0}-{1}", key.Name, programName);
}
result = true;
}
}
catch (SecurityException se)
{
// The user does not have the permissions required to read from the registry key.
logger.Error(se);
}
catch (UnauthorizedAccessException uae)
{
// The user does not have the necessary registry rights.
logger.Error(uae);
}
return(result);
}
static void AddAppsToTar(Stream tar, string apps_directory, bool conventions)
{
string[] directories = Directory.GetDirectories(apps_directory);
foreach (string directory in directories)
{
var app = Path.GetFileName(directory);
Console.WriteLine($"[+] App: {app}");
// manifest comes first
var fManifest = Path.Combine(directory, "_manifest");
using (var fin_manifest = File.OpenRead(fManifest)) {
// AddFileToTarRaw(tarOutputStream, fin_manifest, new FileInfo(fManifest), $"apps/{app}/_manifest");
ABTar.WriteTarFile(app, "", "_manifest", fin_manifest, tar, out var _);
}
// write other files
foreach (var directory2 in Directory.GetDirectories(directory))
{
var domain = Path.GetFileName(directory2);
if (conventions)
{
if (AndroidBackupConventions.TryGetValue(domain, out var dir2Real))
{
Console.WriteLine("[+] {dir2} -> {dir2Real}");
domain = dir2Real;
}
else if (AndroidBackupConventions.ContainsValue(domain))
{
// name accepted
}
else
{
dir2Real = $"{AndroidBackupWildcardDir}/{domain}";
Console.WriteLine("[!] {dir2} -> {dir2Real}");
domain = dir2Real;
}
}
Console.WriteLine($"[+] - {domain}");
AddDirToTar(tar, directory2, app, domain, "", false);
}
}
ABTar.FinishTar(tar);
}
internal bool SetLabel()
{
bool isFileReadOnly = Utils.IsReadOnly(GrepResult);
string basePath = string.IsNullOrWhiteSpace(searchFolderPath) ? string.Empty : searchFolderPath.TrimEnd('\\');
string displayedName = Path.GetFileName(GrepResult.FileNameDisplayed);
if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowFilePathInResults) &&
GrepResult.FileNameDisplayed.Contains(basePath, StringComparison.CurrentCultureIgnoreCase))
{
if (!string.IsNullOrWhiteSpace(basePath))
{
displayedName = GrepResult.FileNameDisplayed.Substring(basePath.Length + 1).TrimStart('\\');
}
else
{
displayedName = GrepResult.FileNameDisplayed;
}
}
if (!string.IsNullOrWhiteSpace(GrepResult.AdditionalInformation))
{
displayedName += " " + GrepResult.AdditionalInformation + " ";
}
int matchCount = GrepResult.Matches == null ? 0 : GrepResult.Matches.Count;
if (matchCount > 0)
{
if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowVerboseMatchCount) && !GrepResult.IsHexFile)
{
var lineCount = GrepResult.Matches.Where(r => r.LineNumber > 0)
.Select(r => r.LineNumber).Distinct().Count();
displayedName = TranslationSource.Format(Resources.Main_ResultList_CountMatchesOnLines, displayedName, matchCount, lineCount);
}
else
{
displayedName = string.Format(Resources.Main_ResultList_CountMatches, displayedName, matchCount);
}
}
if (isFileReadOnly)
{
displayedName = displayedName + " " + Resources.Main_ResultList_ReadOnly;
}
Label = displayedName;
return(isFileReadOnly);
}
internal static void DaemonGet(Library library, string fingerprint)
{
string filename_full = PDFDocumentFileLocations.DocumentPath(library, fingerprint, "pdf");
string filename_short = Path.GetFileName(filename_full);
string pdf_path = IntranetLibraryTools.GetLibraryPDFPath(library.WebLibraryDetail.IntranetPath, filename_short);
DirectoryTools.CreateDirectory(Path.GetDirectoryName(filename_full));
Logging.Info("+Copying down {0}", fingerprint);
File.Copy(pdf_path, filename_full);
Logging.Info("-Copying down {0}", fingerprint);
// Write the audit
if (true)
{
string audit_filename = IntranetLibraryTools.GetLibraryAuditFilename(library.WebLibraryDetail.IntranetPath);
string audit_directory = Path.GetDirectoryName(audit_filename);
if (Directory.Exists(audit_directory))
{
string audit_data = String.Format(
"{0}\t{1}\t{2}\t{3}\t{4}\t{5}\r\n"
, DateTime.UtcNow.ToString("yyyyMMdd.hhmmss")
, ConfigurationManager.Instance.ConfigurationRecord.Account_Username
, ConfigurationManager.Instance.ConfigurationRecord.Account_Nickname
, Environment.UserName
, filename_short
, pdf_path
);
try
{
File.AppendAllText(audit_filename, audit_data);
}
catch (Exception ex)
{
Logging.Warn(ex, "Unable to write intranet sync audit data.");
}
}
}
}
public static async Task <bool> UploadPackagedInis(IEnumerable <Archive> archives)
{
archives = archives.ToArray(); // defensive copy
Utils.Log($"Packaging {archives.Count()} inis");
try
{
await using var ms = new MemoryStream();
using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true))
{
foreach (var e in archives)
{
if (e.State == null)
{
continue;
}
var entry = z.CreateEntry(Path.GetFileName(e.Name));
await using var os = entry.Open();
await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", e.State.GetMetaIni())));
}
}
var client = new Common.Http.Client();
var response = await client.PostAsync($"{Consts.WabbajackBuildServerUri}indexed_files/notify", new ByteArrayContent(ms.ToArray()));
if (response.IsSuccessStatusCode)
{
return(true);
}
Utils.Log("Error sending Inis");
Utils.Log(await response.Content.ReadAsStringAsync());
return(false);
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
return(false);
}
}
private static BrowserEmulationVersion GetBrowserEmulationVersion()
{
BrowserEmulationVersion result;
result = BrowserEmulationVersion.Default;
try
{
RegistryKey key;
key = Registry.CurrentUser.OpenSubKey(BrowserEmulationKey, true);
if (key != null)
{
string programName;
object value;
programName = Path.GetFileName(Environment.GetCommandLineArgs()[0]);
value = key.GetValue(programName, null);
if (value != null)
{
result = (BrowserEmulationVersion)Convert.ToInt32(value);
}
}
}
catch (SecurityException se)
{
// The user does not have the permissions required to read from the registry key.
logger.Error(se);
}
catch (UnauthorizedAccessException uae)
{
// The user does not have the necessary registry rights.
logger.Error(uae);
}
return(result);
}
static void AddDirToTar(Stream tar, string sourceDirectory, string app, string domain, string basePath, bool writeDirEntry = true)
{
// Optionally, write an entry for the directory itself.
if (writeDirEntry)
{
TarEntry tarEntry = TarEntry.CreateEntryFromFile(sourceDirectory);
tarEntry.Name = basePath;
// tarOutputStream.PutNextEntry(tarEntry);
ABTar.WriteTarFile(app, domain, basePath, null, tar, out var _);
}
// Write each file to the tar.
string[] filenames = Directory.GetFiles(sourceDirectory);
foreach (string filename in filenames)
{
using (FileStream file = File.OpenRead(filename)) {
//AddFileToTarRaw(tarOutputStream, inputStream, new FileInfo(filename),
// PathCombineUnixUnsafe(basePath, Path.GetFileName(filename)));
ABTar.WriteTarFile(app, domain,
PathCombineUnixUnsafe(basePath, Path.GetFileName(filename)),
file, tar, out var _);
}
}
// Recurse.
string[] directories = Directory.GetDirectories(sourceDirectory);
foreach (string directory in directories)
{
//AddDirToTar(tarOutputStream, directory,
// PathCombineUnixUnsafe(basePath, Path.GetFileName(directory)));
ABTar.WriteTarFile(app, domain,
PathCombineUnixUnsafe(basePath, Path.GetFileName(directory)),
null, tar, out var _);
}
}
public bool Initialize(string config, string game, out string errorMessage)
{
errorMessage = default;
if (!this._client.CanInitialize(config, game))
{
return(false);
}
this._runValidation = false;
this._basePath = config;
if (!Directory.Exists(this._basePath))
{
errorMessage = $"{this._basePath} does not exist";
this.Log().Error(errorMessage);
return(false);
}
try
{
if (!Directory.Exists(Path.Combine(this._basePath, "BannerLordLauncher Backups")))
{
Directory.CreateDirectory(Path.Combine(this._basePath, "BannerLordLauncher Backups"));
}
}
catch (Exception e)
{
this.Log().Error(e);
}
var launcherData = UserData.Load(this, Path.Combine(this._basePath, "LauncherData.xml")) ?? new UserData();
this._modulePath = Path.Combine(game, "Modules");
this.GameExeFolder = Path.Combine(game, "bin", "Win64_Shipping_Client");
var modulesFolder = Path.Combine(game, "Modules");
if (!Directory.Exists(modulesFolder))
{
errorMessage = $"{modulesFolder} does not exist";
this.Log().Error(errorMessage);
return(false);
}
var modules = Directory.EnumerateDirectories(modulesFolder, "*", SearchOption.TopDirectoryOnly)
.Select(dir => Module.Load(this, Path.GetFileName(dir), game)).Where(module => module != null).ToList();
if (launcherData.SingleplayerData?.ModDatas != null)
{
foreach (var mod in launcherData.SingleplayerData.ModDatas)
{
if (this.Mods.Any(x => x.UserModData.Id.Equals(mod.Id, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
var module = modules.FirstOrDefault(x => x.Id == mod.Id);
if (module == null)
{
this.Log().Warn($"{mod.Id} could not be found in {modulesFolder}");
continue;
}
modules.Remove(module);
var modEntry = new ModEntry {
Module = module, UserModData = mod
};
this.Mods.Add(modEntry);
if (modEntry.Module.Official)
{
modEntry.IsChecked = true;
}
}
}
foreach (var module in modules)
{
if (this.Mods.Any(x => x.Module.Id.Equals(module.Id, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
var modEntry = new ModEntry {
Module = module, UserModData = new UserModData(module.Id, false)
};
this.Mods.Add(modEntry);
if (modEntry.Module.Official)
{
modEntry.IsChecked = true;
}
}
this.AnalyzeAssemblies();
this.BuildDependencies();
this._runValidation = true;
this.Validate();
return(true);
}
private void BackupFiles(IList <IVssWMComponent> components, IDictionary <string, string> volumeMap,
IDictionary <string, string> snapshotVolumeMap, IDictionary <string, string> vmNamesMap,
Options options, ILogger logger)
{
var streams = new List <Stream>();
try
{
foreach (var component in components)
{
string vmBackupPath;
if (options.DirectCopy)
{
vmBackupPath = Path.Combine(options.Output,
string.Format(options.OutputFormat, vmNamesMap[component.ComponentName],
component.ComponentName,
DateTime.Now, ""));
}
else
{
vmBackupPath = Path.Combine(options.Output,
string.Format(options.OutputFormat, vmNamesMap[component.ComponentName],
component.ComponentName,
DateTime.Now,
options.ZipFormat ? ".zip" : ".7z"));
File.Delete(vmBackupPath);
}
var files = new Dictionary <string, Stream>();
foreach (var file in component.Files)
{
string path;
if (file.IsRecursive)
{
path = file.Path;
}
else
{
path = Path.Combine(file.Path, file.FileSpecification);
}
// Get the longest matching path
var volumePath = volumeMap.Keys.OrderBy(o => o.Length).Reverse()
.First(o => path.StartsWith(o, StringComparison.OrdinalIgnoreCase));
var volumeName = volumeMap[volumePath];
// Exclude snapshots
var fileName = Path.GetFileName(path.Substring(volumePath.Length)).ToUpperInvariant();
var include = !path.EndsWith("\\*");
var pathItems = path.Split(Path.DirectorySeparatorChar);
if (pathItems.Length >= 2)
{
if (pathItems[pathItems.Length - 2].ToLowerInvariant() == "snapshots")
{
include = false;
}
}
if (include && options.VhdInclude != null)
{
if (options.VhdInclude.Count(
x => string.CompareOrdinal(x.ToUpperInvariant(), fileName) == 0) == 0)
{
include = false;
}
}
if (include && options.VhdIgnore != null)
{
if (options.VhdIgnore.Count(
x => string.CompareOrdinal(x.ToUpperInvariant(), fileName) == 0) != 0)
{
include = false;
}
}
if (include)
{
if (options.DirectCopy)
{
DoDirectCopy(vmBackupPath, snapshotVolumeMap[volumeName], volumePath.Length, path);
}
else
{
AddPathToCompressionList(files, streams, snapshotVolumeMap[volumeName], volumePath.Length, path);
}
}
else
{
var errorText = $"Ignoring file {path}";
logger.Info(errorText);
Console.WriteLine(errorText);
}
}
if (!options.DirectCopy)
{
logger.Debug($"Start compression. File: {vmBackupPath}");
if (options.ZipFormat)
{
if (options.CompressionLevel == -1)
{
options.CompressionLevel = 6;
}
using (var zf = new ZipFile(vmBackupPath))
{
zf.ParallelDeflateThreshold = -1;
zf.UseZip64WhenSaving = Zip64Option.Always;
zf.Encryption = EncryptionAlgorithm.WinZipAes256;
switch (options.CompressionLevel)
{
case 0:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level0;
break;
case 1:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level1;
break;
case 2:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level2;
break;
case 3:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level3;
break;
case 4:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level4;
break;
case 5:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level5;
break;
case 6:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level6;
break;
case 7:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level7;
break;
case 8:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level8;
break;
case 9:
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.Level9;
break;
}
if (BackupProgress != null)
{
zf.SaveProgress += (sender, e) => ReportZipProgress(component, volumeMap, e);
}
if (!string.IsNullOrEmpty(options.Password))
{
zf.Password = options.Password;
}
foreach (var file in files)
{
logger.Debug($"Adding file: {file.Key}");
zf.AddEntry(file.Key, file.Value);
}
zf.Save();
}
}
else
{
if (options.CompressionLevel == -1)
{
options.CompressionLevel = 3;
}
SevenZipBase.SetLibraryPath(
Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "7z.dll"));
var sevenZip = new SevenZipCompressor
{
ArchiveFormat = OutArchiveFormat.SevenZip,
CompressionMode = CompressionMode.Create,
DirectoryStructure = true,
PreserveDirectoryRoot = false
};
if (options.MultiThreaded)
{
sevenZip.CustomParameters.Add("mt", "on");
}
sevenZip.CustomParameters.Add("d", "24");
switch (options.CompressionLevel)
{
case 0:
sevenZip.CompressionLevel = CompressionLevel.None;
break;
case 1:
case 2:
sevenZip.CompressionLevel = CompressionLevel.Fast;
break;
case 3:
case 4:
case 5:
sevenZip.CompressionLevel = CompressionLevel.Low;
break;
case 6:
sevenZip.CompressionLevel = CompressionLevel.Normal;
break;
case 7:
case 8:
sevenZip.CompressionLevel = CompressionLevel.High;
break;
case 9:
sevenZip.CompressionLevel = CompressionLevel.Ultra;
break;
}
if (BackupProgress != null)
{
sevenZip.FileCompressionStarted += (sender, e) =>
{
var ebp = new BackupProgressEventArgs
{
AcrhiveFileName = e.FileName,
Action = EventAction.StartingArchive
};
_currentFile = e.FileName;
Report7ZipProgress(component, volumeMap, ebp);
if (_cancel)
{
e.Cancel = true;
}
};
sevenZip.FileCompressionFinished += (sender, e) =>
{
var ebp = new BackupProgressEventArgs
{
AcrhiveFileName = _currentFile,
Action = EventAction.ArchiveDone
};
_currentFile = string.Empty;
Report7ZipProgress(component, volumeMap, ebp);
};
sevenZip.Compressing += (sender, e) =>
{
var ebp = new BackupProgressEventArgs
{
AcrhiveFileName = _currentFile,
Action = EventAction.PercentProgress,
CurrentEntry = _currentFile,
PercentDone = e.PercentDone
};
Report7ZipProgress(component, volumeMap, ebp);
};
}
if (string.IsNullOrEmpty(options.Password))
{
sevenZip.CompressStreamDictionary(files, vmBackupPath);
}
else
{
sevenZip.CompressStreamDictionary(files, vmBackupPath, options.Password);
}
}
logger.Debug("Compression finished");
if (_cancel)
{
if (File.Exists(vmBackupPath))
{
File.Delete(vmBackupPath);
}
throw new BackupCancelledException();
}
}
}
}
finally
{
// Make sure that all streams are closed
foreach (var s in streams)
{
s.Close();
}
}
}
public FormattedGrepResult(GrepSearchResult result, string folderPath)
{
GrepResult = result;
bool isFileReadOnly = Utils.IsReadOnly(GrepResult);
bool isSuccess = GrepResult.IsSuccess;
string basePath = string.IsNullOrWhiteSpace(folderPath) ? string.Empty : folderPath.TrimEnd('\\');
string displayedName = Path.GetFileName(GrepResult.FileNameDisplayed);
if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowFilePathInResults) &&
GrepResult.FileNameDisplayed.Contains(basePath, StringComparison.CurrentCultureIgnoreCase))
{
if (!string.IsNullOrWhiteSpace(basePath))
{
displayedName = GrepResult.FileNameDisplayed.Substring(basePath.Length + 1).TrimStart('\\');
}
else
{
displayedName = GrepResult.FileNameDisplayed;
}
}
if (!string.IsNullOrWhiteSpace(GrepResult.AdditionalInformation))
{
displayedName += " " + GrepResult.AdditionalInformation + " ";
}
int matchCount = (GrepResult.Matches == null ? 0 : GrepResult.Matches.Count);
if (matchCount > 0)
{
if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowVerboseMatchCount))
{
var lineCount = GrepResult.Matches.Where(r => r.LineNumber > 0)
.Select(r => r.LineNumber).Distinct().Count();
displayedName = string.Format("{0} ({1} matches on {2} lines)", displayedName, matchCount, lineCount);
}
else
{
displayedName = string.Format("{0} ({1})", displayedName, matchCount);
}
}
if (isFileReadOnly)
{
result.ReadOnly = true;
displayedName = displayedName + " [read-only]";
}
Label = displayedName;
if (isFileReadOnly)
{
Style = "ReadOnly";
}
if (!isSuccess)
{
Style = "Error";
}
FormattedLines = new LazyResultsList(result, this);
FormattedLines.LineNumberColumnWidthChanged += FormattedLines_PropertyChanged;
FormattedLines.LoadFinished += FormattedLines_LoadFinished;
}
public static Task <string> UploadFile(WorkQueue queue, string filename, Action <double> progressFn)
{
var tcs = new TaskCompletionSource <string>();
Task.Run(async() =>
{
var client = GetAuthorizedClient();
var fsize = new FileInfo(filename).Length;
var hash_task = filename.FileHashAsync();
var response = await client.PutAsync(UploadURL + $"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
tcs.SetException(new Exception($"Start Error: {response.StatusCode} {response.ReasonPhrase}"));
return;
}
IEnumerable <long> Blocks(long fsize)
{
for (long block = 0; block *BLOCK_SIZE < fsize; block++)
{
yield return(block);
}
}
var key = await response.Content.ReadAsStringAsync();
long sent = 0;
using (var iqueue = new WorkQueue(MAX_CONNECTIONS))
{
iqueue.Report("Starting Upload", Percent.One);
await Blocks(fsize)
.PMap(iqueue, async block_idx =>
{
if (tcs.Task.IsFaulted)
{
return;
}
var block_offset = block_idx * BLOCK_SIZE;
var block_size = block_offset + BLOCK_SIZE > fsize
? fsize - block_offset
: BLOCK_SIZE;
Interlocked.Add(ref sent, block_size);
progressFn((double)sent / fsize);
int retries = 0;
using (var fs = File.OpenRead(filename))
{
fs.Position = block_offset;
var data = new byte[block_size];
await fs.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetException(new Exception($"Put Error: {response.StatusCode} {response.ReasonPhrase}"));
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
{
tcs.SetResult($"Sync Error {val} vs {block_offset + data.Length}");
tcs.SetException(new Exception($"Sync Error {val} vs {block_offset + data.Length}"));
}
}
});
}
if (!tcs.Task.IsFaulted)
{
progressFn(1.0);
var hash = (await hash_task).FromBase64().ToHex();
response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent(""));
if (response.IsSuccessStatusCode)
{
tcs.SetResult(await response.Content.ReadAsStringAsync());
}
else
{
tcs.SetException(new Exception($"Finalization Error: {response.StatusCode} {response.ReasonPhrase}"));
}
}
progressFn(0.0);
});
return(tcs.Task);
}
// *************************************************************************************************************
// *** MIGRATION TO OPEN SOURCE CODE ***************************************************************************
// *************************************************************************************************************
private void AddLegacyWebLibrariesThatCanBeFoundOnDisk()
{
/**
* Plan:
* Iterate through all the folders in the Qiqqa data directory
* If a folder contains a valid Library record and it is a WEB library, then add it to our list with the word '[LEGACY]' in front of it
*/
string base_directory_path = UpgradePaths.V037To038.SQLiteUpgrade.BaseDirectoryForQiqqa;
Logging.Info("Going to scan for web libraries at: {0}", base_directory_path);
if (Directory.Exists(base_directory_path))
{
string[] library_directories = Directory.GetDirectories(base_directory_path);
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 1 : Web & Known Libraries", library_directory);
string databaselist_file = Path.GetFullPath(Path.Combine(library_directory, @"Qiqqa.known_web_libraries"));
if (File.Exists(databaselist_file))
{
LoadKnownWebLibraries(databaselist_file, true);
}
}
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 2 : Intranet Libraries", library_directory);
string databaselist_file = IntranetLibraryTools.GetLibraryDetailPath(library_directory);
if (File.Exists(databaselist_file))
{
IntranetLibraryDetail intranet_library_detail = IntranetLibraryDetail.Read(databaselist_file);
UpdateKnownWebLibraryFromIntranet(library_directory, extra_info_message_on_skip: String.Format(" as obtained from file {0}", databaselist_file));
}
}
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 3 : Bundles", library_directory);
// must be a qiqqa_bundle and/or qiqqa_bundle_manifest file set
Logging.Warn("Auto bundle import at startup is not yet suppoerted.");
}
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 4 : Local and Legacy Libraries", library_directory);
string database_file = Path.GetFullPath(Path.Combine(library_directory, @"Qiqqa.library"));
if (File.Exists(database_file))
{
var library_id = Path.GetFileName(library_directory);
WebLibraryDetail new_web_library_detail = new WebLibraryDetail();
new_web_library_detail.Id = library_id;
new_web_library_detail.Title = "Legacy Web Library - " + new_web_library_detail.Id;
new_web_library_detail.IsReadOnly = false;
// library: UNKNOWN type
UpdateKnownWebLibrary(new_web_library_detail);
}
}
}
}
public AppCompatCache(string filename, int controlSet, bool noLogs)
{
byte[] rawBytes = null;
Caches = new List <IAppCompatCache>();
var controlSetIds = new List <int>();
RegistryKey subKey = null;
var isLiveRegistry = string.IsNullOrEmpty(filename);
if (isLiveRegistry)
{
var keyCurrUser = Microsoft.Win32.Registry.LocalMachine;
var subKey2 = keyCurrUser.OpenSubKey(@"SYSTEM\CurrentControlSet\Control\Session Manager\AppCompatCache");
if (subKey2 == null)
{
subKey2 =
keyCurrUser.OpenSubKey(@"SYSTEM\CurrentControlSet\Control\Session Manager\AppCompatibility");
if (subKey2 == null)
{
Console.WriteLine(
@"'CurrentControlSet\Control\Session Manager\AppCompatCache' key not found! Exiting");
return;
}
}
rawBytes = (byte[])subKey2.GetValue("AppCompatCache", null);
subKey2 = keyCurrUser.OpenSubKey(@"SYSTEM\Select");
ControlSet = (int)subKey2.GetValue("Current");
var is32Bit = Is32Bit(filename, null);
var cache = Init(rawBytes, is32Bit, ControlSet);
Caches.Add(cache);
return;
}
RegistryHive reg;
Privilege[] privileges = { Privilege.EnableDelegation, Privilege.Impersonate, Privilege.Tcb };
using (new PrivilegeEnabler(Privilege.Backup, privileges))
{
ControlSet = controlSet;
if (File.Exists(filename) == false && RawCopy.Helper.RawFileExists(filename) == false)
{
throw new FileNotFoundException($"File not found ({filename})!");
}
var dirname = Path.GetDirectoryName(filename);
var hiveBase = Path.GetFileName(filename);
List <RawCopy.RawCopyReturn> rawFiles = null;
try
{
reg = new RegistryHive(filename)
{
RecoverDeleted = true
};
}
catch (IOException)
{
//file is in use
if (RawCopy.Helper.IsAdministrator() == false)
{
throw new UnauthorizedAccessException("Administrator privileges not found!");
}
_logger.Warn($"'{filename}' is in use. Rerouting...\r\n");
var files = new List <string>();
files.Add(filename);
var logFiles = Directory.GetFiles(dirname, $"{hiveBase}.LOG?").ToList();
var log1 = $"{dirname}\\{hiveBase}.LOG1";
var log2 = $"{dirname}\\{hiveBase}.LOG2";
if (logFiles.Count == 0)
{
if (RawCopy.Helper.RawFileExists(log1))
{
logFiles.Add(log1);
}
if (RawCopy.Helper.RawFileExists(log2))
{
logFiles.Add(log2);
}
}
foreach (var logFile in logFiles)
{
files.Add(logFile);
}
rawFiles = RawCopy.Helper.GetFiles(files);
reg = new RegistryHive(rawFiles.First().FileBytes, rawFiles.First().InputFilename);
}
if (reg.Header.PrimarySequenceNumber != reg.Header.SecondarySequenceNumber)
{
if (string.IsNullOrEmpty(dirname))
{
dirname = ".";
}
var logFiles = Directory.GetFiles(dirname, $"{hiveBase}.LOG?").ToList();
var log1 = $"{dirname}\\{hiveBase}.LOG1";
var log2 = $"{dirname}\\{hiveBase}.LOG2";
if (logFiles.Count == 0)
{
if (File.Exists(log1))
{
logFiles.Add(log1);
}
if (File.Exists(log2))
{
logFiles.Add(log2);
}
}
if (logFiles.Count == 0)
{
if (RawCopy.Helper.RawFileExists(log1))
{
logFiles.Add(log1);
}
if (RawCopy.Helper.RawFileExists(log2))
{
logFiles.Add(log2);
}
}
if (logFiles.Count == 0)
{
if (noLogs == false)
{
_logger.Warn("Registry hive is dirty and no transaction logs were found in the same directory! LOGs should have same base name as the hive. Aborting!!");
throw new Exception("Sequence numbers do not match and transaction logs were not found in the same directory as the hive. Aborting");
}
else
{
_logger.Warn("Registry hive is dirty and no transaction logs were found in the same directory. Data may be missing! Continuing anyways...");
}
}
else
{
if (noLogs == false)
{
if (rawFiles != null)
{
var lt = new List <TransactionLogFileInfo>();
foreach (var rawCopyReturn in rawFiles.Skip(1).ToList())
{
var tt = new TransactionLogFileInfo(rawCopyReturn.InputFilename, rawCopyReturn.FileBytes);
lt.Add(tt);
}
reg.ProcessTransactionLogs(lt, true);
}
else
{
reg.ProcessTransactionLogs(logFiles.ToList(), true);
}
}
else
{
_logger.Warn("Registry hive is dirty and transaction logs were found in the same directory, but --nl was provided. Data may be missing! Continuing anyways...");
}
}
}
reg.ParseHive();
}
if (controlSet == -1)
{
for (var i = 0; i < 10; i++)
{
subKey = reg.GetKey($@"ControlSet00{i}\Control\Session Manager\AppCompatCache");
if (subKey == null)
{
subKey = reg.GetKey($@"ControlSet00{i}\Control\Session Manager\AppCompatibility");
}
if (subKey != null)
{
controlSetIds.Add(i);
}
}
if (controlSetIds.Count > 1)
{
_logger.Warn(
$"***The following ControlSet00x keys will be exported: {string.Join(",", controlSetIds)}. Use -c to process keys individually\r\n");
}
}
else
{
//a control set was passed in
subKey = reg.GetKey($@"ControlSet00{ControlSet}\Control\Session Manager\AppCompatCache");
if (subKey == null)
{
subKey = reg.GetKey($@"ControlSet00{ControlSet}\Control\Session Manager\AppCompatibility");
}
if (subKey == null)
{
throw new Exception($"Could not find ControlSet00{ControlSet}. Exiting");
}
controlSetIds.Add(ControlSet);
}
var is32 = Is32Bit(filename, reg);
_logger.Debug($@"**** Found {controlSetIds.Count} ids to process");
foreach (var id in controlSetIds)
{
_logger.Debug($@"**** Processing id {id}");
// var hive2 = new RegistryHiveOnDemand(filename);
subKey = reg.GetKey($@"ControlSet00{id}\Control\Session Manager\AppCompatCache");
if (subKey == null)
{
_logger.Debug($@"**** Initial subkey null, getting appCompatability key");
subKey = reg.GetKey($@"ControlSet00{id}\Control\Session Manager\AppCompatibility");
}
_logger.Debug($@"**** Looking AppCompatcache value");
var val = subKey?.Values.SingleOrDefault(c => c.ValueName == "AppCompatCache");
if (val != null)
{
_logger.Debug($@"**** Found AppCompatcache value");
rawBytes = val.ValueDataRaw;
}
if (rawBytes == null)
{
_logger.Error($@"'AppCompatCache' value not found for 'ControlSet00{id}'! Exiting");
}
var cache = Init(rawBytes, is32, id);
Caches.Add(cache);
}
}
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path,
string rel_path, bool topLevel)
{
var hash = abs_path.FileHash();
var fi = new FileInfo(abs_path);
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(abs_path))
{
var result = await TryGetContentsFromServer(hash);
if (result != null)
{
Utils.Log($"Downloaded VFS data for {Path.GetFileName(abs_path)}");
VirtualFile Convert(IndexedVirtualFile file, string path, VirtualFile vparent)
{
var vself = new VirtualFile
{
Context = context,
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = fi.LastWriteTimeUtc.Ticks,
LastAnalyzed = DateTime.Now.Ticks,
Hash = file.Hash,
};
vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();
return(vself);
}
return(Convert(result, rel_path, parent));
}
}
var self = new VirtualFile
{
Context = context,
Name = rel_path,
Parent = parent,
Size = fi.Length,
LastModified = fi.LastWriteTimeUtc.Ticks,
LastAnalyzed = DateTime.Now.Ticks,
Hash = hash
};
if (context.UseExtendedHashes)
{
self.ExtendedHashes = ExtendedHashes.FromFile(abs_path);
}
if (FileExtractor.CanExtract(abs_path))
{
using (var tempFolder = context.GetTemporaryFolder())
{
await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName);
var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
.PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName), false));
self.Children = list.ToImmutableList();
}
}
return(self);
}
public FormattedGrepResult(GrepSearchResult result, string folderPath)
{
grepResult = result;
fileInfo = new FileInfo(grepResult.FileNameReal);
bool isFileReadOnly = Utils.IsReadOnly(grepResult);
bool isSuccess = grepResult.IsSuccess;
string basePath = string.IsNullOrWhiteSpace(folderPath) ? string.Empty :
Utils.GetBaseFolder(folderPath).TrimEnd('\\');
string displayedName = Path.GetFileName(grepResult.FileNameDisplayed);
if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowFilePathInResults) &&
grepResult.FileNameDisplayed.Contains(basePath))
{
if (!string.IsNullOrWhiteSpace(basePath))
{
displayedName = grepResult.FileNameDisplayed.Substring(basePath.Length + 1).TrimStart('\\');
}
else
{
displayedName = grepResult.FileNameDisplayed;
}
}
if (!string.IsNullOrWhiteSpace(grepResult.AdditionalInformation))
{
displayedName += " " + grepResult.AdditionalInformation + " ";
}
int matchCount = (grepResult.Matches == null ? 0 : grepResult.Matches.Count);
if (matchCount > 0)
{
if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ShowVerboseMatchCount))
{
var lineCount = grepResult.Matches.Where(r => r.LineNumber > 0)
.Select(r => r.LineNumber).Distinct().Count();
displayedName = string.Format("{0} ({1} matches on {2} lines)", displayedName, matchCount, lineCount);
}
else
{
displayedName = string.Format("{0} ({1})", displayedName, matchCount);
}
}
if (isFileReadOnly)
{
result.ReadOnly = true;
displayedName = displayedName + " [read-only]";
}
label = displayedName;
if (isFileReadOnly)
{
style = "ReadOnly";
}
if (!isSuccess)
{
style = "Error";
}
formattedLines = new LazyResultsList(result, this);
formattedLines.LineNumberColumnWidthChanged += formattedLines_PropertyChanged;
formattedLines.LoadFinished += formattedLines_LoadFinished;
if (GrepSettings.Instance.Get <bool>(GrepSettings.Key.ExpandResults))
{
IsExpanded = true;
}
}
// *************************************************************************************************************
// *** MIGRATION TO OPEN SOURCE CODE ***************************************************************************
// *************************************************************************************************************
private void AddLegacyWebLibrariesThatCanBeFoundOnDisk()
{
WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
try
{
ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(AddLegacyWebLibrariesThatCanBeFoundOnDisk));
/**
* Plan:
* - Iterate through all the folders in the Qiqqa data directory.
* - If a folder contains a valid Library record and it is a WEB library,
* then add it to our list with the word '[LEGACY]' in front of it.
*/
string base_directory_path = UpgradePaths.V037To038.SQLiteUpgrade.BaseDirectoryForQiqqa;
Logging.Info("Going to scan for web libraries at: {0}", base_directory_path);
if (Directory.Exists(base_directory_path))
{
string[] library_directories = Directory.GetDirectories(base_directory_path);
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 1 : Web & Known Libraries", library_directory);
string databaselist_file = Path.GetFullPath(Path.Combine(library_directory, @"Qiqqa.known_web_libraries"));
if (File.Exists(databaselist_file))
{
LoadKnownWebLibraries(databaselist_file, only_load_those_libraries_which_are_actually_present: true);
}
}
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 2 : Intranet Libraries", library_directory);
string databaselist_file = IntranetLibraryTools.GetLibraryDetailPath(library_directory);
if (File.Exists(databaselist_file))
{
IntranetLibraryDetail intranet_library_detail = IntranetLibraryDetail.Read(databaselist_file);
UpdateKnownWebLibraryFromIntranet(library_directory, extra_info_message_on_skip: String.Format(" as obtained from file {0}", databaselist_file));
}
}
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 3 : Bundles", library_directory);
// must be a qiqqa_bundle and/or qiqqa_bundle_manifest file set
Logging.Warn("Auto bundle import at startup is not yet supported.");
}
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 4 : Local and Legacy Libraries", library_directory);
string database_file = LibraryDB.GetLibraryDBPath(library_directory);
string db_syncref_path = IntranetLibraryTools.GetLibraryMetadataPath(library_directory);
// add/update only if this is not a Internet sync directory/DB!
if (File.Exists(db_syncref_path))
{
Logging.Info("Skip the Qiqqa Internet/Intranet Sync directory and the sync DB contained therein: '{0}'", db_syncref_path);
// https://github.com/jimmejardine/qiqqa-open-source/issues/145 :: delete lib file when it is very small and was illegally
// constructed by a previous v82beta Qiqqa release:
if (File.Exists(database_file))
{
long s3length = File.GetSize(database_file);
if (6 * 1024 > s3length)
{
Logging.Warn("DELETE the wrongfully created DB file '{0}' in the Qiqqa Internet/Intranet Sync directory and the sync DB contained therein: '{1}', which has precedence!", database_file, db_syncref_path);
FileTools.DeleteToRecycleBin(database_file);
}
else
{
Logging.Error("Inspect the Library DB file '{0}' in the Qiqqa Internet/Intranet Sync directory and the sync DB contained therein: '{1}', which MAY have precedence. Delete one of these manually to clean up your system as Qiqqa heuristics cannot tell which is the prevalent metadata database here!", database_file, db_syncref_path);
}
}
continue;
}
if (File.Exists(database_file))
{
var library_id = Path.GetFileName(library_directory);
WebLibraryDetail new_web_library_detail = new WebLibraryDetail();
new_web_library_detail.Id = library_id;
new_web_library_detail.Title = "Legacy Web Library - " + new_web_library_detail.Id;
new_web_library_detail.IsReadOnly = false;
// library: UNKNOWN type
UpdateKnownWebLibrary(new_web_library_detail);
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem while scanning for (legacy) libraries.");
}
}
private void BackupFiles(IList <IVssWMComponent> components, IDictionary <string, string> volumeMap,
IDictionary <string, string> snapshotVolumeMap, IDictionary <string, string> vmNamesMap,
Options options)
{
IList <System.IO.Stream> streams = new List <System.IO.Stream>();
try
{
foreach (var component in components)
{
string vmBackupPath;
if (options.DirectCopy)
{
vmBackupPath = Path.Combine(options.Output,
string.Format(options.OutputFormat, vmNamesMap[component.ComponentName],
component.ComponentName,
DateTime.Now, ""));
}
else
{
vmBackupPath = Path.Combine(options.Output,
string.Format(options.OutputFormat, vmNamesMap[component.ComponentName],
component.ComponentName,
DateTime.Now,
options.ZipFormat ? ".zip" : ".7z"));
File.Delete(vmBackupPath);
}
var files = new Dictionary <string, System.IO.Stream>();
foreach (var file in component.Files)
{
string path;
if (file.IsRecursive)
{
path = file.Path;
}
else
{
path = Path.Combine(file.Path, file.FileSpecification);
}
// Get the longest matching path
var volumePath = volumeMap.Keys.OrderBy(o => o.Length).Reverse().First(o => path.StartsWith(o, StringComparison.OrdinalIgnoreCase));
var volumeName = volumeMap[volumePath];
// Exclude snapshots
var fileName = Path.GetFileName(path.Substring(volumePath.Length)).ToUpperInvariant();
var include = !path.EndsWith("\\*");
var pathItems = path.Split(Path.DirectorySeparatorChar);
if (pathItems.Length >= 2)
{
if (pathItems[pathItems.Length - 2].ToLowerInvariant() == "snapshots")
{
include = false;
}
}
if (include && options.VhdInclude != null)
{
if (options.VhdInclude.Count(x => string.CompareOrdinal(x.ToUpperInvariant(), fileName) == 0) == 0)
{
include = false;
}
}
if (include && options.VhdIgnore != null)
{
if (options.VhdIgnore.Count(x => string.CompareOrdinal(x.ToUpperInvariant(), fileName) == 0) != 0)
{
include = false;
}
}
if (include)
{
if (options.DirectCopy)
{
DoDirectCopy(vmBackupPath, snapshotVolumeMap[volumeName], volumePath.Length, path);
}
else
{
AddPathToSevenZip(files, streams, snapshotVolumeMap[volumeName], volumePath.Length, path);
}
}
else
{
Console.WriteLine("Ignoring file {0}", path);
}
}
if (!options.DirectCopy)
{
SevenZipBase.SetLibraryPath(Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "7z.dll"));
var sevenZip = new SevenZipCompressor
{
ArchiveFormat = options.ZipFormat ? OutArchiveFormat.Zip : OutArchiveFormat.SevenZip,
CompressionMode = CompressionMode.Create,
DirectoryStructure = true,
PreserveDirectoryRoot = false
};
sevenZip.CustomParameters.Add("mt", "on");
switch (options.CompressionLevel)
{
case 0:
sevenZip.CompressionLevel = CompressionLevel.None;
break;
case 1:
sevenZip.CompressionLevel = CompressionLevel.Fast;
break;
case 2:
sevenZip.CompressionLevel = CompressionLevel.Fast;
break;
case 3:
sevenZip.CompressionLevel = CompressionLevel.Low;
break;
case 4:
sevenZip.CompressionLevel = CompressionLevel.Low;
break;
case 5:
sevenZip.CompressionLevel = CompressionLevel.Low;
break;
case 6:
sevenZip.CompressionLevel = CompressionLevel.Normal;
break;
case 7:
sevenZip.CompressionLevel = CompressionLevel.High;
break;
case 8:
sevenZip.CompressionLevel = CompressionLevel.High;
break;
case 9:
sevenZip.CompressionLevel = CompressionLevel.Ultra;
break;
}
if (BackupProgress != null)
{
sevenZip.FileCompressionStarted += (sender, e) =>
{
var ebp = new BackupProgressEventArgs()
{
AcrhiveFileName = e.FileName,
Action = EventAction.StartingArchive
};
_sevenZipCurrentFile = e.FileName;
Report7ZipProgress(component, volumeMap, ebp);
if (_cancel)
{
e.Cancel = true;
}
};
sevenZip.FileCompressionFinished += (sender, e) =>
{
var ebp = new BackupProgressEventArgs
{
AcrhiveFileName = _sevenZipCurrentFile,
Action = EventAction.ArchiveDone
};
_sevenZipCurrentFile = string.Empty;
Report7ZipProgress(component, volumeMap, ebp);
};
sevenZip.Compressing += (sender, e) =>
{
var ebp = new BackupProgressEventArgs
{
AcrhiveFileName = _sevenZipCurrentFile,
Action = EventAction.PercentProgress,
CurrentEntry = _sevenZipCurrentFile,
PercentDone = e.PercentDone
};
Report7ZipProgress(component, volumeMap, ebp);
if (_cancel)
{
e.Cancel = true;
}
};
}
if (string.IsNullOrEmpty(options.Password))
{
sevenZip.CompressStreamDictionary(files, vmBackupPath);
}
else
{
sevenZip.CompressStreamDictionary(files, vmBackupPath, options.Password);
}
if (_cancel)
{
if (File.Exists(vmBackupPath))
{
File.Delete(vmBackupPath);
}
throw new BackupCancelledException();
}
}
}
}
finally
{
// Make sure that all streams are closed
foreach (var s in streams)
{
s.Close();
}
}
}
private string MkLegalSizedPath(string basename, string typeIdStr)
{
const int PATH_MAX = 240; // must be less than 255 / 260 - see also https://kb.acronis.com/content/39790
string root = Path.GetDirectoryName(basename);
string name = Path.GetFileName(basename);
string dataname = Path.GetFileNameWithoutExtension(DataFile);
string ext = SubStr(Path.GetExtension(DataFile), 1).Trim(); // produce the extension without leading dot
if (ext.StartsWith("bib"))
{
ext = SubStr(ext, 3).Trim();
}
if (ext.Length > 0)
{
ext = "." + ext;
}
// UNC long filename/path support by forcing this to be a UNC path:
string filenamebase = $"{dataname}.{name}{ext}{ExtensionWithDot}";
// first make the full path without the approved/received, so that that bit doesn't make a difference
// in the length check and subsequent decision to produce a shorthand filename path or not:
// It's not always needed, but do the different shorthand conversions anyway and pick the longest fitting one:
string short_tn = SanitizeFilename(CamelCaseShorthand(name));
string short_dn = SanitizeFilename(SubStr(dataname, 0, 10) + CamelCaseShorthand(dataname));
string hash = StreamMD5.FromText(filenamebase).ToUpper();
string short_hash = SubStr(hash, 0, Math.Max(6, 11 - short_tn.Length));
// this variant will fit in the length criterium, guaranteed:
string alt_filepath0 = Path.GetFullPath(Path.Combine(root, $"{short_dn}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}"));
string filepath = alt_filepath0;
// next, we construct the longer variants to check if they fit.
//
// DO NOTE that we create a path without typeIdStr part first, because we want both received and approved files to be based
// on the *same* alt selection decision!
string picked_alt_filepath = Path.GetFullPath(Path.Combine(root, $"{short_dn}.{short_hash}_{short_tn}{ext}.APPROVEDXYZ{ExtensionWithDot}"));
name = SanitizeFilename(name);
dataname = SanitizeFilename(dataname);
string alt_filepath1 = Path.GetFullPath(Path.Combine(root, $"{short_dn}_{short_hash}.{name}{ext}.APPROVEDXYZ{ExtensionWithDot}"));
if (alt_filepath1.Length < PATH_MAX)
{
filepath = Path.GetFullPath(Path.Combine(root, $"{short_dn}_{short_hash}.{name}{ext}{typeIdStr}{ExtensionWithDot}"));
picked_alt_filepath = alt_filepath1;
}
// second alternative: only pick this one if it fits and produces a longer name:
string alt_filepath2 = Path.GetFullPath(Path.Combine(root, $"{dataname}.{short_hash}_{short_tn}{ext}.APPROVEDXYZ{ExtensionWithDot}"));
if (alt_filepath2.Length < PATH_MAX && alt_filepath2.Length > picked_alt_filepath.Length)
{
filepath = Path.GetFullPath(Path.Combine(root, $"{dataname}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}"));
picked_alt_filepath = alt_filepath2;
}
else
{
// third alt: the 'optimally trimmed' test name used as part of the filename:
int trim_length = PATH_MAX - alt_filepath0.Length + 10 - 1;
string short_dn2 = SanitizeFilename(SubStr(dataname, 0, trim_length) + CamelCaseShorthand(dataname));
string alt_filepath3 = Path.GetFullPath(Path.Combine(root, $"{short_dn2}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}"));
if (alt_filepath3.Length < PATH_MAX && alt_filepath3.Length > picked_alt_filepath.Length)
{
filepath = Path.GetFullPath(Path.Combine(root, $"{short_dn2}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}"));
picked_alt_filepath = alt_filepath3;
}
}
// fourth alt: the full, unadulterated path; if it fits in the length criterium, take it anyway
string alt_filepath4 = Path.GetFullPath(Path.Combine(root, $"{dataname}.{name}{ext}.APPROVEDXYZ{ExtensionWithDot}"));
if (alt_filepath4.Length < PATH_MAX)
{
// UNC long filename/path support by forcing this to be a UNC path:
filepath = Path.GetFullPath(Path.Combine(root, $"{dataname}.{name}{ext}{typeIdStr}{ExtensionWithDot}"));
picked_alt_filepath = alt_filepath4;
}
return(filepath);
}
public override async Task <JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
{
using (var queue = new WorkQueue(4))
{
Utils.Log($"Indexing game files");
var states = GameRegistry.Games.Values
.Where(game => game.GameLocation() != null && game.MainExecutable != null)
.SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories)
.Select(file => new GameFileSourceDownloader.State
{
Game = game.Game,
GameVersion = game.InstalledVersion,
GameFile = file.RelativeTo(game.GameLocation()),
}))
.ToList();
var pks = states.Select(s => s.PrimaryKeyString).Distinct().ToArray();
Utils.Log($"Found {pks.Length} archives to cross-reference with the database");
var found = (await db.DownloadStates
.AsQueryable().Where(s => pks.Contains(s.Key))
.Select(s => s.Key)
.ToListAsync())
.ToDictionary(s => s);
states = states.Where(s => !found.ContainsKey(s.PrimaryKeyString)).ToList();
Utils.Log($"Found {states.Count} archives to index");
await states.PMap(queue, async state =>
{
var path = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile);
Utils.Log($"Hashing Game file {path}");
try
{
state.Hash = await path.FileHashAsync();
}
catch (IOException)
{
Utils.Log($"Unable to hash {path}");
}
});
var with_hash = states.Where(state => state.Hash != null).ToList();
Utils.Log($"Inserting {with_hash.Count} jobs.");
var jobs = states.Select(state => new IndexJob {
Archive = new Archive {
Name = Path.GetFileName(state.GameFile), State = state
}
})
.Select(j => new Job {
Payload = j, RequiresNexus = j.UsesNexus
})
.ToList();
if (jobs.Count > 0)
{
await db.Jobs.InsertManyAsync(jobs);
}
return(JobResult.Success());
}
}
