public void CopyDataTo(Stream output)
{
var bw = new BinaryWriter(output);
WriteHeader(bw);
using (var fs = File.OpenRead(_bsa._filename))
using (var br = new BinaryReader(fs))
{
foreach (var chunk in _chunks)
{
var full = new byte[chunk._fullSz];
var isCompressed = chunk._packSz != 0;
br.BaseStream.Seek((long)chunk._offset, SeekOrigin.Begin);
if (!isCompressed)
{
br.BaseStream.Read(full, 0, full.Length);
}
else
{
byte[] compressed = new byte[chunk._packSz];
br.BaseStream.Read(compressed, 0, compressed.Length);
var inflater = new Inflater();
inflater.SetInput(compressed);
inflater.Inflate(full);
}
bw.BaseStream.Write(full, 0, full.Length);
}
}
}
private static void AddPathToSevenZip(Dictionary <string, System.IO.Stream> files, IList <System.IO.Stream> streams, string snapshotPath, int volumePathLength, string vmPath)
{
var srcPath = Path.Combine(snapshotPath, vmPath.Substring(volumePathLength));
if (Directory.Exists(srcPath))
{
foreach (var srcChildPath in Directory.GetFileSystemEntries(srcPath))
{
var srcChildPathRel = srcChildPath.Substring(snapshotPath.EndsWith(Path.PathSeparator.ToString(), StringComparison.CurrentCultureIgnoreCase) ? snapshotPath.Length : snapshotPath.Length + 1);
var childPath = Path.Combine(vmPath.Substring(0, volumePathLength), srcChildPathRel);
AddPathToSevenZip(files, streams, snapshotPath, volumePathLength, childPath);
}
}
else if (File.Exists(srcPath))
{
var s = File.OpenRead(srcPath);
files.Add(vmPath.Substring(volumePathLength), s);
streams.Add(s);
}
else
{
var lowerPath = srcPath.ToLowerInvariant();
var isIgnorable = lowerPath.EndsWith(".avhdx") || lowerPath.EndsWith(".vmrs") ||
lowerPath.EndsWith(".bin") || lowerPath.EndsWith(".vsv");
if (!isIgnorable)
{
throw new Exception($"Entry \"{srcPath}\" not found in snapshot");
}
}
}
public void CopyDataTo(Stream output)
{
using (var in_file = File.OpenRead(_bsa._fileName))
using (var rdr = new BinaryReader(in_file))
{
rdr.BaseStream.Position = _dataOffset;
if (_bsa.HeaderType == VersionType.SSE)
{
if (Compressed)
{
var r = LZ4Stream.Decode(rdr.BaseStream);
r.CopyToLimit(output, (int)_originalSize);
}
else
{
rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize);
}
}
else
{
if (Compressed)
{
using (var z = new InflaterInputStream(rdr.BaseStream))
{
z.CopyToLimit(output, (int)_originalSize);
}
}
else
{
rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize);
}
}
}
}
public static string FromFile(string filename)
{
//using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 1024 * 1024))
using (FileStream fs = File.OpenRead(filename))
{
return(FromStream(fs));
}
}
private static T ProtoLoad_NotRedundant <T>(string filename)
{
using (FileStream fs = File.OpenRead(filename))
{
T animal = Serializer.Deserialize <T>(fs);
return(animal);
}
}
public static async Task CopyFileAsync(string sourcePath, string destinationPath)
{
using (Stream source = File.OpenRead(sourcePath))
{
using (Stream destination = File.Create(destinationPath))
{
await source.CopyToAsync(destination);
}
}
}
/// <summary>
/// MurMur3 hashes the file pointed to by this string
/// </summary>
/// <param name="file"></param>
/// <returns></returns>
public static string FileSHA256(this string file)
{
var sha = new SHA256Managed();
using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write))
{
using (var i = File.OpenRead(file))
i.CopyToWithStatus(new FileInfo(file).Length, o, $"Hashing {Path.GetFileName(file)}");
}
return(sha.Hash.ToBase64());
}
public static T FromBSON <T>(this string filename, bool root_is_array = false)
{
using (var fo = File.OpenRead(filename))
using (var br = new BsonDataReader(fo, readRootValueAsArray: root_is_array, DateTimeKind.Local))
{
var serializer = JsonSerializer.Create(new JsonSerializerSettings()
{
TypeNameHandling = TypeNameHandling.Auto
});
return(serializer.Deserialize <T>(br));
}
}
private void LoadWord_LOCK(WordInWordIndex wiwi)
{
// If the word is already loaded, nothing to do...
if (wiwi.IsLoaded)
{
return;
}
try
{
string filename = Filename_GangList(wiwi.WordId);
using (FileStream fs = File.OpenRead(filename))
{
{
bool gang_has_corrupted_word_counts = false;
List <WordEntry> word_entrys = Serializer.Deserialize <List <WordEntry> >(fs);
int gang_start = GangStart(wiwi.WordId);
for (int i = 0; i < word_entrys.Count; ++i)
{
if (0 != String.Compare(word_in_word_indexes[gang_start + i].Word, word_entrys[i].Word))
{
throw new Exception("The ordering of the word index is corrupt: words don't match");
}
if (null != word_in_word_indexes[gang_start + i].DocumentIds)
{
Logging.Warn("The ordering of the word index is corrupt: document_ids should be null");
}
WordInWordIndex wiwi_just_loaded = word_in_word_indexes[gang_start + i];
bool corruption_detected = wiwi_just_loaded.SetDocumentIds(word_entrys[i].DocumentIds, word_entrys[i].DocumentIdsCount);
if (corruption_detected)
{
gang_has_corrupted_word_counts = true;
}
}
if (gang_has_corrupted_word_counts)
{
Logging.Warn("The ordering of a word index in the gang is corrupt: doc counts don't match (the user probably exited before the gang was saved...)");
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem loading the word document list for word {0}:{1}. Assuming it was empty.", wiwi.WordId, wiwi.Word);
bool set_result = wiwi.SetDocumentIds(new List <int>(), new List <int>());
}
}
private void ReadMasterList()
{
Logging.Info("+ReadMasterList");
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (locker)
{
l1_clk.LockPerfTimerStop();
try
{
using (FileStream fs = File.OpenRead(GetFilename_MasterList()))
{
Headers headers = Serializer.Deserialize <Headers>(fs);
// First the documents
{
foreach (var header in headers.documents)
{
fingerprint_to_document_ids[header.Fingerprint] = header.DocumentId;
document_id_to_fingerprints[header.DocumentId] = header.Fingerprint;
}
}
// Then the words
{
foreach (var header in headers.words)
{
WordInWordIndex wiwi = new WordInWordIndex(header.Word, header.WordId, header.DocCount);
// Sanity check that they are in the right order
if (wiwi.WordId != word_in_word_indexes.Count)
{
throw new Exception("The ordering of the word index is corrupt");
}
// Add to our maps
word_in_word_indexes.Add(wiwi);
word_in_word_index_lookups[wiwi.Word] = wiwi;
}
}
}
}
catch (Exception ex)
{
Logging.Warn(ex, "Unable to load index master list, so starting from scratch");
}
}
Logging.Info("-ReadMasterList");
}
public static string FileHash(this string file)
{
var hash = new xxHashConfig();
hash.HashSizeInBits = 64;
hash.Seed = 0x42;
using (var fs = File.OpenRead(file))
{
var config = new xxHashConfig();
config.HashSizeInBits = 64;
var value = xxHashFactory.Instance.Create(config).ComputeHash(fs);
return(value.AsBase64String());
}
}
static void AddAppsToTar(Stream tar, string apps_directory, bool conventions)
{
string[] directories = Directory.GetDirectories(apps_directory);
foreach (string directory in directories)
{
var app = Path.GetFileName(directory);
Console.WriteLine($"[+] App: {app}");
// manifest comes first
var fManifest = Path.Combine(directory, "_manifest");
using (var fin_manifest = File.OpenRead(fManifest)) {
// AddFileToTarRaw(tarOutputStream, fin_manifest, new FileInfo(fManifest), $"apps/{app}/_manifest");
ABTar.WriteTarFile(app, "", "_manifest", fin_manifest, tar, out var _);
}
// write other files
foreach (var directory2 in Directory.GetDirectories(directory))
{
var domain = Path.GetFileName(directory2);
if (conventions)
{
if (AndroidBackupConventions.TryGetValue(domain, out var dir2Real))
{
Console.WriteLine("[+] {dir2} -> {dir2Real}");
domain = dir2Real;
}
else if (AndroidBackupConventions.ContainsValue(domain))
{
// name accepted
}
else
{
dir2Real = $"{AndroidBackupWildcardDir}/{domain}";
Console.WriteLine("[!] {dir2} -> {dir2Real}");
domain = dir2Real;
}
}
Console.WriteLine($"[+] - {domain}");
AddDirToTar(tar, directory2, app, domain, "", false);
}
}
ABTar.FinishTar(tar);
}
private static void DoDirectCopy(string vmBackupPath, string snapshotPath, int volumePathLength, string vmPath)
{
var srcPath = Path.Combine(snapshotPath, vmPath.Substring(volumePathLength));
if (Directory.Exists(srcPath))
{
foreach (var srcChildPath in Directory.GetFileSystemEntries(srcPath))
{
var srcChildPathRel =
srcChildPath.Substring(snapshotPath.EndsWith(Path.PathSeparator.ToString(),
StringComparison.CurrentCultureIgnoreCase)
? snapshotPath.Length
: snapshotPath.Length + 1);
var childPath = Path.Combine(vmPath.Substring(0, volumePathLength), srcChildPathRel);
DoDirectCopy(vmBackupPath, snapshotPath, volumePathLength, childPath);
}
}
else if (File.Exists(srcPath))
{
var outputName = Path.Combine(vmBackupPath, vmPath.Substring(volumePathLength));
using (var s = File.OpenRead(srcPath))
{
var folder = Path.GetDirectoryName(outputName);
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
using (var ns = File.Create(outputName))
{
s.CopyTo(ns);
}
}
}
else
{
var lowerPath = srcPath.ToLowerInvariant();
var isIgnorable = lowerPath.EndsWith(".avhdx") || lowerPath.EndsWith(".vmrs") ||
lowerPath.EndsWith(".bin") || lowerPath.EndsWith(".vsv");
if (!isIgnorable)
{
throw new Exception($"Entry \"{srcPath}\" not found in snapshot");
}
}
}
private void LoadFromDisk()
{
try
{
HashIndex = new Dictionary <string, IEnumerable <VirtualFile> >();
Utils.Log("Loading VFS Cache");
if (!File.Exists("vfs_cache.bin"))
{
return;
}
_files = new Dictionary <string, VirtualFile>();
try
{
using (var fs = File.OpenRead("vfs_cache.bin"))
using (var br = new BinaryReader(fs))
{
var magic = Encoding.ASCII.GetString(br.ReadBytes(Magic.Length));
if (magic != Magic || br.ReadUInt64() != FileVersion)
{
fs.Close();
File.Delete("vfs_cache.bin");
return;
}
while (true)
{
var fr = VirtualFile.Read(br);
_files.Add(fr.FullPath, fr);
}
}
}
catch (EndOfStreamException)
{
}
CleanDB();
}
catch (Exception ex)
{
Utils.Log($"Purging cache due to {ex}");
File.Delete("vfs_cache.bson");
_files.Clear();
}
}
private void LoadOriginalSize()
{
using (var in_file = File.OpenRead(_bsa._fileName))
using (var rdr = new BinaryReader(in_file))
{
rdr.BaseStream.Position = _offset;
string _name;
int file_size = _size;
if (_bsa.HasNameBlobs)
{
var name_size = rdr.ReadByte();
file_size -= name_size + 1;
rdr.BaseStream.Position = _offset + 1 + name_size;
}
_originalSize = rdr.ReadUInt32();
}
}
private const uint CKM_Magic = 0x52415442; // BTAR
private async Task ConvertCKMToZip(string src, string dest)
{
using var reader = new BinaryReader(File.OpenRead(src));
var magic = reader.ReadUInt32();
if (magic != CKM_Magic)
{
throw new InvalidDataException("Invalid magic format in CKM parsing");
}
ushort majorVersion = reader.ReadUInt16();
ushort minorVersion = reader.ReadUInt16();
if (majorVersion != 1)
{
throw new InvalidDataException("Archive major version is unknown. Should be 1.");
}
if (minorVersion < 2 || minorVersion > 4)
{
throw new InvalidDataException("Archive minor version is unknown. Should be 2, 3, or 4.");
}
await using var fos = File.Create(dest);
using var archive = new ZipArchive(fos, ZipArchiveMode.Create);
while (reader.PeekChar() != -1)
{
ushort nameLength = reader.ReadUInt16();
string name = Encoding.UTF8.GetString(reader.ReadBytes(nameLength));
ulong dataLength = reader.ReadUInt64();
if (dataLength > int.MaxValue)
{
throw new Exception();
}
var entry = archive.CreateEntry(name, CompressionLevel.NoCompression);
await using var output = entry.Open();
await reader.BaseStream.CopyToLimitAsync(output, (long)dataLength);
}
}
internal List <string> GetArchiveEntryNames(VirtualFile file)
{
if (!file.IsStaged)
{
throw new InvalidDataException("File is not staged");
}
if (file.Extension == ".bsa")
{
using (var ar = new BSAReader(file.StagedPath))
{
return(ar.Files.Select(f => f.Path).ToList());
}
}
if (file.Extension == ".zip")
{
using (var s = new ZipFile(File.OpenRead(file.StagedPath)))
{
s.IsStreamOwner = true;
s.UseZip64 = UseZip64.On;
if (s.OfType <ZipEntry>().FirstOrDefault(e => !e.CanDecompress) == null)
{
return(s.OfType <ZipEntry>()
.Where(f => f.IsFile)
.Select(f => f.Name.Replace('/', '\\'))
.ToList());
}
}
}
/*
* using (var e = new ArchiveFile(file.StagedPath))
* {
* return e.Entries
* .Where(f => !f.IsFolder)
* .Select(f => f.FileName).ToList();
* }*/
return(null);
}
public static HashResponse GetMD5HashResponseFromFile(string filename, int bytesToHash)
{
var hashResponse = new HashResponse();
try
{
using (Stream stream = File.OpenRead(filename))
{
var buf = new byte[bytesToHash];
int bytesRead = stream.Read(buf, 0, buf.Length);
long totalBytesRead = bytesRead;
while (bytesRead > 0 && totalBytesRead <= bytesToHash)
{
bytesRead = stream.Read(buf, 0, buf.Length);
totalBytesRead += bytesRead;
}
hashResponse.BytesHashed = totalBytesRead;
hashResponse.IsPartialHash = stream.Length > bytesToHash;
using (var md5 = MD5.Create())
{
hashResponse.Hash = md5.ComputeHash(buf);
}
}
return(hashResponse);
}
catch (FileLoadException) // if doing hasing on system drive cant open files dont care.
{
return(null);
}
catch (Exception ex)
{
ILogger logger = new Logger();
logger.LogDebug(
$" original filename \"{filename}\"");
logger.LogException(ex, "MD5Hash");
return(null);
}
}
private void LoadFromDisk()
{
try
{
Utils.Log("Loading VFS Cache");
if (!File.Exists("vfs_cache.bin"))
{
return;
}
_files = new Dictionary <string, VirtualFile>();
try
{
using (var fs = File.OpenRead("vfs_cache.bin"))
using (var br = new BinaryReader(fs))
{
while (true)
{
var fr = VirtualFile.Read(br);
_files.Add(fr.FullPath, fr);
}
}
}
catch (EndOfStreamException ex)
{
}
CleanDB();
}
catch (Exception ex)
{
Utils.Log($"Purging cache due to {ex}");
File.Delete("vfs_cache.bson");
_isDirty = true;
_files.Clear();
}
}
public static HashResponse GetMD5HashFromFile(string filename)
{
try
{
using (Stream stream = File.OpenRead(filename))
{
var hashResponse = new HashResponse();
using (var md5 = MD5.Create())
{
hashResponse.Hash = md5.ComputeHash(stream);
hashResponse.IsPartialHash = false;
hashResponse.BytesHashed += stream.Length;
return(hashResponse);
}
}
}
catch (Exception ex)
{
ILogger logger = new Logger();
logger.LogException(ex, "MD5Hash");
return(null);
}
}
public void CopyDataTo(Stream output)
{
using (var fs = File.OpenRead(_bsa._filename))
{
fs.Seek((long)_offset, SeekOrigin.Begin);
uint len = Compressed ? _size : _realSize;
var bytes = new byte[len];
fs.Read(bytes, 0, (int)len);
if (!Compressed)
{
output.Write(bytes, 0, bytes.Length);
}
else
{
var uncompressed = new byte[_realSize];
var inflater = new Inflater();
inflater.SetInput(bytes);
inflater.Inflate(uncompressed);
output.Write(uncompressed, 0, uncompressed.Length);
}
}
}
static void AddDirToTar(Stream tar, string sourceDirectory, string app, string domain, string basePath, bool writeDirEntry = true)
{
// Optionally, write an entry for the directory itself.
if (writeDirEntry)
{
TarEntry tarEntry = TarEntry.CreateEntryFromFile(sourceDirectory);
tarEntry.Name = basePath;
// tarOutputStream.PutNextEntry(tarEntry);
ABTar.WriteTarFile(app, domain, basePath, null, tar, out var _);
}
// Write each file to the tar.
string[] filenames = Directory.GetFiles(sourceDirectory);
foreach (string filename in filenames)
{
using (FileStream file = File.OpenRead(filename)) {
//AddFileToTarRaw(tarOutputStream, inputStream, new FileInfo(filename),
// PathCombineUnixUnsafe(basePath, Path.GetFileName(filename)));
ABTar.WriteTarFile(app, domain,
PathCombineUnixUnsafe(basePath, Path.GetFileName(filename)),
file, tar, out var _);
}
}
// Recurse.
string[] directories = Directory.GetDirectories(sourceDirectory);
foreach (string directory in directories)
{
//AddDirToTar(tarOutputStream, directory,
// PathCombineUnixUnsafe(basePath, Path.GetFileName(directory)));
ABTar.WriteTarFile(app, domain,
PathCombineUnixUnsafe(basePath, Path.GetFileName(directory)),
null, tar, out var _);
}
}
public static void FurtherDiagnoseDBProblem(Exception ex, List <Exception> corruption_list, string library_path)
{
SQLiteException sql_ex = ex as SQLiteException;
// so we had a failure (or several)... now check the state of the database file and report on our findings:
StringBuilder sb = new StringBuilder();
do
{
sb.AppendLine("--- Diagnosis for reported problem ---");
sb.AppendLine("======================================");
sb.AppendLine("");
if (!File.Exists(library_path))
{
sb.AppendLine($"--> The database file '{library_path}' does not exist.");
break;
}
bool looks_sane = true;
bool is_readonly = false;
// what are the access rights and size?
try
{
var info = File.GetFileSystemEntryInfo(library_path);
sb.AppendLine($"--> File Attributes: {info.Attributes}");
sb.AppendLine($"--> File Creation Date (UTC): {info.CreationTimeUtc}");
sb.AppendLine($"--> File Last Changed Date (UTC): {info.LastWriteTimeUtc}");
sb.AppendLine($"--> File Last Access Date (UTC): {info.LastAccessTimeUtc}");
sb.AppendLine($"--> Is marked as READ ONLY: {info.IsReadOnly}");
sb.AppendLine($"--> Is marked as OFFLINE: {info.IsOffline}");
sb.AppendLine($"--> Is marked as archived: {info.IsArchive}");
sb.AppendLine($"--> Is marked as HIDDEN: {info.IsHidden}");
sb.AppendLine($"--> Is a SYSTEM FILE: {info.IsSystem}");
sb.AppendLine($"--> Is encrypted by the operating system: {info.IsEncrypted}");
sb.AppendLine($"--> Is compressed by the operating system: {info.IsCompressed}");
sb.AppendLine($"--> Is a mount point: {info.IsMountPoint}");
sb.AppendLine($"--> Is temporary: {info.IsTemporary}");
sb.AppendLine($"--> Is a symbolic link: {info.IsSymbolicLink}");
sb.AppendLine($"--> Is a sparse file: {info.IsSparseFile}");
sb.AppendLine($"--> Is a reparse point: {info.IsReparsePoint}");
sb.AppendLine($"--> Is not content indexed by the operating system: {info.IsNotContentIndexed}");
sb.AppendLine($"--> Is a directory: {info.IsDirectory}");
sb.AppendLine($"--> Is a device: {info.IsDevice}");
sb.AppendLine($"--> Is a normal file: {info.IsNormal}");
sb.AppendLine($"--> File size: {info.FileSize} bytes");
is_readonly = info.IsReadOnly;
if (info.IsOffline || info.IsHidden || info.IsSystem || info.IsEncrypted || info.IsMountPoint || info.IsTemporary || info.IsSymbolicLink || info.IsSparseFile || info.IsReparsePoint || info.IsDirectory || info.IsDevice)
{
sb.AppendLine("");
sb.AppendLine("--> WARNING: this doesn't look like a very normal file.");
sb.AppendLine(" Check the attributes above to determine if they are as you expect.");
sb.AppendLine("");
looks_sane = false;
}
}
catch (Exception ex2)
{
sb.AppendLine($"--> FAILED to collect the file attributes: {ex2.ToStringAllExceptionDetails()}");
looks_sane = false;
}
// Check if we can open the file for basic I/O:
try
{
// read the entire file into a 1M buffer. Watch for errors.
byte[] buf = new byte[1024 * 1024];
const int count = 1024 * 1024;
long length_read = 0;
using (var stream = File.OpenRead(library_path))
{
int rc;
while (true)
{
rc = stream.Read(buf, 0, count);
if (rc > 0)
{
length_read += rc;
}
else if (rc == 0)
{
// EOF
break;
}
else
{
throw new Exception($"stream.Read produced a negative result: {rc}");
}
}
}
long file_size = File.GetSize(library_path);
if (file_size != length_read)
{
throw new Exception($"stream.Read was unable to read/scan the entire file:\n file size reported by the file system = {file_size} bytes, length scanned = {length_read} bytes");
}
sb.AppendLine($"--> Successfully scanned the entire file: length scanned = {length_read} bytes");
}
catch (Exception ex2)
{
sb.AppendLine($"--> FAILED to read/scan the file: {ex2.ToStringAllExceptionDetails()}");
looks_sane = false;
}
if (!is_readonly)
{
// check if we can open the file for WRITE access
try
{
using (var stream = File.OpenWrite(library_path))
{
sb.AppendLine($"--> Successfully opened the file for WRITE ACCESS");
}
}
catch (Exception ex2)
{
sb.AppendLine($"--> FAILED to open the file for WRITE ACCESS:);");
sb.AppendLine(ex2.ToStringAllExceptionDetails());
looks_sane = false;
}
}
if (corruption_list != null && corruption_list.Count > 0)
{
if (looks_sane)
{
sb.AppendLine("--> WARNING: while the RAW file scan and access checks may have PASSED,");
sb.AppendLine(" the Qiqqa inner systems certainly found stuff in the file to complain about:");
sb.AppendLine(" these data corruptions (a.k.a. DECODE FAILURES) have already been reported,");
sb.AppendLine(" but here they are once more in summarized form:");
}
else
{
sb.AppendLine("--> ERROR: while the RAW file scan and access checks may have FAILED,");
sb.AppendLine(" the Qiqqa inner systems also found stuff in the file to complain about");
sb.AppendLine(" -- which is VERY PROBABLY related to or caused by the findings reported above.");
sb.AppendLine("");
sb.AppendLine(" The data corruptions (a.k.a. DECODE FAILURES) have already been reported,");
sb.AppendLine(" but here they are once more in summarized form:");
}
sb.AppendLine("");
int index = 1;
foreach (var corruption in corruption_list)
{
sb.AppendLine($" #{index.ToString("03")}: {corruption.Message.Split('\n')[0]}");
index++;
}
sb.AppendLine($" --- {corruption_list.Count} reported data corruptions ---------------------------------------");
looks_sane = false;
}
if (sql_ex != null)
{
sb.AppendLine("");
sb.AppendLine(" As this report is about a SQLite database error, it MAY be useful to search");
sb.AppendLine(" the Internet for generic help and/or discussion of the reported SQLite error:");
sb.AppendLine("");
int errorcode = sql_ex.ErrorCode;
int basenum = errorcode & 0xFF;
int extended_shift = errorcode >> 8;
int herr = sql_ex.HResult;
sb.AppendLine(" ( ref: https://sqlite.org/rescode.html )");
sb.AppendLine(" ( ref: https://sqlite.org/c3ref/c_abort.html )");
sb.AppendLine($" SQLite Error Code: {basenum}");
if (extended_shift != 0)
{
sb.AppendLine(" ( ref: https://sqlite.org/c3ref/c_abort_rollback.html )");
sb.AppendLine($" SQLite Extended Error Code: ({basenum} | ({extended_shift} << 8)) = {errorcode}");
}
else
{
sb.AppendLine(" Reported error code is NOT a SQLite Extended Error Code.");
}
sb.AppendLine($" SQLite HResult: {herr.ToString("08:X")}");
sb.AppendLine("");
sb.AppendLine($" SQLite: the define constants (i.e. compile-time options): {SQLiteConnection.DefineConstants}");
sb.AppendLine($" SQLite: the underlying SQLite core library: {SQLiteConnection.SQLiteVersion}");
sb.AppendLine($" SQLite: SQLITE_SOURCE_ID: {SQLiteConnection.SQLiteSourceId}");
sb.AppendLine($" SQLite: the compile-time options: {SQLiteConnection.SQLiteCompileOptions}");
sb.AppendLine($" SQLite: the version of the interop SQLite assembly: {SQLiteConnection.InteropVersion}");
sb.AppendLine($" SQLite: the unique identifier for the source checkout used to build the interop assembly: {SQLiteConnection.InteropSourceId}");
sb.AppendLine($" SQLite: the compile-time options used to compile the SQLite interop assembly: {SQLiteConnection.InteropCompileOptions}");
sb.AppendLine($" SQLite: the version of the managed components: {SQLiteConnection.ProviderVersion}");
sb.AppendLine($" SQLite: the unique identifier for the source checkout used to build the managed components: {SQLiteConnection.ProviderSourceId}");
sb.AppendLine($" SQLite: the extra connection flags: {SQLiteConnection.SharedFlags}");
sb.AppendLine($" SQLite: the default connection flags: {SQLiteConnection.DefaultFlags}");
sb.AppendLine("");
sb.AppendLine(" (ref: https://sqlite.org/c3ref/extended_result_codes.html )");
sb.AppendLine(" SQLite Extended Error Reporting has been ENABLED: SetExtendedResultCodes(true)");
}
sb.AppendLine("---------");
if (looks_sane)
{
sb.AppendLine("");
sb.AppendLine("--> VERDICT OK(?): this DOES look like a very normal file.");
sb.AppendLine("");
sb.AppendLine(" HOWEVER, it may have incorrect a.k.a. 'corrupted' content, which made Qiqqa barf,");
sb.AppendLine(" or there's something else going on which this simply diagnosis routine");
sb.AppendLine(" is unable to unearth.");
sb.AppendLine("");
sb.AppendLine(" Please file a report at the Qiqqa issue tracker and include this logging");
sb.AppendLine(" for further inspection.");
}
else
{
sb.AppendLine("");
sb.AppendLine("--> VERDICT BAD(?): as far as this simple diagnostic routine can tell,");
sb.AppendLine(" this is NOT an 'okay' file.");
sb.AppendLine("");
if (is_readonly)
{
sb.AppendLine(" It MAY be marked READ-ONLY, which MAY be okay in your book, but is certainly");
sb.AppendLine(" unexpected here.");
sb.AppendLine("");
}
sb.AppendLine(" There's something going on which this simply diagnosis routine CANNOT diagnose further.");
sb.AppendLine("");
sb.AppendLine(" Please file a report at the Qiqqa issue tracker and include this logging");
sb.AppendLine(" for further inspection.");
}
sb.AppendLine("");
sb.AppendLine("==================== End of diagnostics report ============================================");
} while (false);
Logging.Error(sb.ToString());
}
public BSAReader(string filename) : this(File.OpenRead(filename))
{
_fileName = filename;
}
public static Task <string> UploadFile(WorkQueue queue, string filename)
{
var tcs = new TaskCompletionSource <string>();
queue.QueueTask(async() =>
{
var client = new HttpClient();
var fsize = new FileInfo(filename).Length;
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var response = await client.PutAsync(UploadURL + $"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var key = await response.Content.ReadAsStringAsync();
using (var iqueue = new WorkQueue(8))
{
await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE))
.PMap(iqueue, async block_idx =>
{
var block_offset = block_idx * BLOCK_SIZE;
var block_size = block_offset + BLOCK_SIZE > fsize
? fsize - block_offset
: BLOCK_SIZE;
using (var fs = File.OpenRead(filename))
{
fs.Position = block_offset;
var data = new byte[block_size];
await fs.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
}
});
}
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
if (response.IsSuccessStatusCode)
{
tcs.SetResult(await response.Content.ReadAsStringAsync());
}
else
{
tcs.SetResult("FAILED");
}
});
return(tcs.Task);
}
public override Stream OpenRead(string path)
{
return(AfsFile.OpenRead(path));
}
static int Main(string[] args)
{
return(Parser.Default.ParseArguments <UnpackOptions, ShowOptions, PackOptions>(args)
.MapResult(
(UnpackOptions opts) => {
if (opts.extractTarOnly)
{
using (var inflater = GetTarInflaterInputStream(File.OpenRead(opts.file))) {
using (var tar = File.Create(Path.Combine(opts.directory, "backup.abh.tar"))) {
inflater.CopyTo(tar);
}
}
}
else
{
using (var tarStream = GetTarInputStream(File.OpenRead(opts.file))) {
ExtractTarByEntry(tarStream, opts.directory);
}
// tarArchive.ExtractContents(opts.directory);
}
return 0;
},
(ShowOptions opts) => {
using (var tarStream = GetTarInputStream(File.OpenRead(opts.file))) {
TarEntry tarEntry;
while ((tarEntry = tarStream.GetNextEntry()) != null)
{
var entryStr = tarEntry.IsDirectory ? " [D] " : " [F] ";
entryStr += $"{tarEntry.Name} {tarEntry.Size:n0} bytes";
Console.WriteLine(entryStr);
}
}
return 0;
},
(PackOptions opts) => {
var tarname = $"{opts.file}.tar";
using (var tar = File.Create(tarname)) {
AddAppsToTar(tar, opts.apps_dir, !opts.disableConventions);
}
using (var outAB = File.OpenWrite(opts.file)) {
outputAndroidBackupHeader(outAB);
outAB.WriteByte(0x78);
outAB.WriteByte(0xDA);
var chksum = new Adler32();
using (var fTar = File.OpenRead(tarname)) {
using (var br = new BinaryReader(fTar, Encoding.UTF8, true)) {
var BUFLEN = 4096;
while (true)
{
var buf = br.ReadBytes(BUFLEN);
if (buf.Length <= 0)
{
break;
}
chksum.Update(buf);
}
}
fTar.Seek(0, SeekOrigin.Begin);
using (var defOut = new DeflateStream(outAB, CompressionMode.Compress, true)) {
fTar.CopyTo(defOut);
}
}
outAB.Write(BitConverter.GetBytes((uint)chksum.Value), 0, 4);
}
return 0;
},
errs => 1));
}
private async Task CreateMetaFiles()
{
Utils.Log("Getting Nexus API key, please click authorize if a browser window appears");
var nexusClient = await NexusApiClient.Get();
var archives = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f =>
File.Exists(f) && Path.GetExtension(f) != Consts.MetaFileExtension && Path.GetExtension(f) != Consts.HashFileExtension &&
!File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)));
await archives.PMap(Queue, async f =>
{
Info($"Creating meta file for {Path.GetFileName(f)}");
var metaString = "[General]\n" +
"repository=Nexus\n" +
$"gameName={GameName}\n";
string hash;
using (var md5 = MD5.Create())
using (var stream = File.OpenRead(f))
{
Utils.Log($"Calculating hash for {Path.GetFileName(f)}");
var cH = md5.ComputeHash(stream);
hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant();
Utils.Log($"Hash is {hash}");
}
var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
if (md5Response.Count >= 1)
{
var modInfo = md5Response[0].mod;
metaString += $"modID={modInfo.mod_id}\n" +
$"modName={modInfo.name}\n" +
$"fileID={md5Response[0].file_details.file_id}\n" +
$"version={md5Response[0].file_details.version}\n" +
$"hash={hash}\n";
File.WriteAllText(f + Consts.MetaFileExtension, metaString, Encoding.UTF8);
}
else
{
Error("Error while getting information from NexusMods via MD5 hash!");
}
});
var otherFiles = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f =>
Path.GetExtension(f) == Consts.MetaFileExtension && !ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)));
await otherFiles.PMap(Queue, async f =>
{
Info($"File {f} is not in ActiveArchives");
var lines = File.ReadAllLines(f);
if (lines.Length == 0 || !lines.Any(line => lines.Contains("directURL=")))
{
if (lines.Length == 0)
{
return;
}
lines.Do(line =>
{
var tag = "";
if (line.Contains("tag="))
{
tag = line.Substring("tag=".Length);
}
if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
{
return;
}
Info($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
});
}
else
{
Info($"File {f} appears to not be from the Nexus, adding to ActiveArchives");
ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
}
});
Info("Checking for Steam Workshop Items...");
if (!_isSteamGame || _steamGame == null || !_hasSteamWorkshopItems)
{
return;
}
_steamGame.WorkshopItems.Do(item =>
{
var filePath = Path.Combine(DownloadsFolder, $"steamWorkshopItem_{item.ItemID}.meta");
if (File.Exists(filePath))
{
Utils.Log($"File {filePath} already exists, skipping");
return;
}
Utils.Log($"Creating meta file for {item.ItemID}");
var metaString = "[General]\n" +
"repository=Steam\n" +
$"gameName={GameName}\n" +
$"steamID={_steamGame.ID}\n" +
$"itemID={item.ItemID}\n" +
$"itemSize={item.Size}\n";
try
{
File.WriteAllText(filePath, metaString);
}
catch (Exception e)
{
Utils.Error(e, $"Exception while writing to disk at {filePath}");
}
});
}
public static Task <string> UploadFile(WorkQueue queue, string filename, Action <double> progressFn)
{
var tcs = new TaskCompletionSource <string>();
Task.Run(async() =>
{
var client = GetAuthorizedClient();
var fsize = new FileInfo(filename).Length;
var hash_task = filename.FileHashAsync();
var response = await client.PutAsync(UploadURL + $"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
tcs.SetException(new Exception($"Start Error: {response.StatusCode} {response.ReasonPhrase}"));
return;
}
IEnumerable <long> Blocks(long fsize)
{
for (long block = 0; block *BLOCK_SIZE < fsize; block++)
{
yield return(block);
}
}
var key = await response.Content.ReadAsStringAsync();
long sent = 0;
using (var iqueue = new WorkQueue(MAX_CONNECTIONS))
{
iqueue.Report("Starting Upload", Percent.One);
await Blocks(fsize)
.PMap(iqueue, async block_idx =>
{
if (tcs.Task.IsFaulted)
{
return;
}
var block_offset = block_idx * BLOCK_SIZE;
var block_size = block_offset + BLOCK_SIZE > fsize
? fsize - block_offset
: BLOCK_SIZE;
Interlocked.Add(ref sent, block_size);
progressFn((double)sent / fsize);
int retries = 0;
using (var fs = File.OpenRead(filename))
{
fs.Position = block_offset;
var data = new byte[block_size];
await fs.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetException(new Exception($"Put Error: {response.StatusCode} {response.ReasonPhrase}"));
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
{
tcs.SetResult($"Sync Error {val} vs {block_offset + data.Length}");
tcs.SetException(new Exception($"Sync Error {val} vs {block_offset + data.Length}"));
}
}
});
}
if (!tcs.Task.IsFaulted)
{
progressFn(1.0);
var hash = (await hash_task).FromBase64().ToHex();
response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent(""));
if (response.IsSuccessStatusCode)
{
tcs.SetResult(await response.Content.ReadAsStringAsync());
}
else
{
tcs.SetException(new Exception($"Finalization Error: {response.StatusCode} {response.ReasonPhrase}"));
}
}
progressFn(0.0);
});
return(tcs.Task);
}