// *************************************************************************************************************
// *** MIGRATION TO OPEN SOURCE CODE ***************************************************************************
// *************************************************************************************************************
private void AddLegacyWebLibrariesThatCanBeFoundOnDisk()
{
WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
try
{
ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(AddLegacyWebLibrariesThatCanBeFoundOnDisk));
/**
* Plan:
* - Iterate through all the folders in the Qiqqa data directory.
* - If a folder contains a valid Library record and it is a WEB library,
* then add it to our list with the word '[LEGACY]' in front of it.
*/
string base_directory_path = UpgradePaths.V037To038.SQLiteUpgrade.BaseDirectoryForQiqqa;
Logging.Info("Going to scan for web libraries at: {0}", base_directory_path);
if (Directory.Exists(base_directory_path))
{
string[] library_directories = Directory.GetDirectories(base_directory_path);
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 1 : Web & Known Libraries", library_directory);
string databaselist_file = Path.GetFullPath(Path.Combine(library_directory, @"Qiqqa.known_web_libraries"));
if (File.Exists(databaselist_file))
{
LoadKnownWebLibraries(databaselist_file, only_load_those_libraries_which_are_actually_present: true);
}
}
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 2 : Intranet Libraries", library_directory);
string databaselist_file = IntranetLibraryTools.GetLibraryDetailPath(library_directory);
if (File.Exists(databaselist_file))
{
IntranetLibraryDetail intranet_library_detail = IntranetLibraryDetail.Read(databaselist_file);
UpdateKnownWebLibraryFromIntranet(library_directory, extra_info_message_on_skip: String.Format(" as obtained from file {0}", databaselist_file));
}
}
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 3 : Bundles", library_directory);
// must be a qiqqa_bundle and/or qiqqa_bundle_manifest file set
Logging.Warn("Auto bundle import at startup is not yet supported.");
}
foreach (string library_directory in library_directories)
{
Logging.Info("Inspecting directory {0} - Phase 4 : Local and Legacy Libraries", library_directory);
string database_file = LibraryDB.GetLibraryDBPath(library_directory);
string db_syncref_path = IntranetLibraryTools.GetLibraryMetadataPath(library_directory);
// add/update only if this is not a Internet sync directory/DB!
if (File.Exists(db_syncref_path))
{
Logging.Info("Skip the Qiqqa Internet/Intranet Sync directory and the sync DB contained therein: '{0}'", db_syncref_path);
// https://github.com/jimmejardine/qiqqa-open-source/issues/145 :: delete lib file when it is very small and was illegally
// constructed by a previous v82beta Qiqqa release:
if (File.Exists(database_file))
{
long s3length = File.GetSize(database_file);
if (6 * 1024 > s3length)
{
Logging.Warn("DELETE the wrongfully created DB file '{0}' in the Qiqqa Internet/Intranet Sync directory and the sync DB contained therein: '{1}', which has precedence!", database_file, db_syncref_path);
FileTools.DeleteToRecycleBin(database_file);
}
else
{
Logging.Error("Inspect the Library DB file '{0}' in the Qiqqa Internet/Intranet Sync directory and the sync DB contained therein: '{1}', which MAY have precedence. Delete one of these manually to clean up your system as Qiqqa heuristics cannot tell which is the prevalent metadata database here!", database_file, db_syncref_path);
}
}
continue;
}
if (File.Exists(database_file))
{
var library_id = Path.GetFileName(library_directory);
WebLibraryDetail new_web_library_detail = new WebLibraryDetail();
new_web_library_detail.Id = library_id;
new_web_library_detail.Title = "Legacy Web Library - " + new_web_library_detail.Id;
new_web_library_detail.IsReadOnly = false;
// library: UNKNOWN type
UpdateKnownWebLibrary(new_web_library_detail);
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem while scanning for (legacy) libraries.");
}
}
public static void FurtherDiagnoseDBProblem(Exception ex, List <Exception> corruption_list, string library_path)
{
SQLiteException sql_ex = ex as SQLiteException;
// so we had a failure (or several)... now check the state of the database file and report on our findings:
StringBuilder sb = new StringBuilder();
do
{
sb.AppendLine("--- Diagnosis for reported problem ---");
sb.AppendLine("======================================");
sb.AppendLine("");
if (!File.Exists(library_path))
{
sb.AppendLine($"--> The database file '{library_path}' does not exist.");
break;
}
bool looks_sane = true;
bool is_readonly = false;
// what are the access rights and size?
try
{
var info = File.GetFileSystemEntryInfo(library_path);
sb.AppendLine($"--> File Attributes: {info.Attributes}");
sb.AppendLine($"--> File Creation Date (UTC): {info.CreationTimeUtc}");
sb.AppendLine($"--> File Last Changed Date (UTC): {info.LastWriteTimeUtc}");
sb.AppendLine($"--> File Last Access Date (UTC): {info.LastAccessTimeUtc}");
sb.AppendLine($"--> Is marked as READ ONLY: {info.IsReadOnly}");
sb.AppendLine($"--> Is marked as OFFLINE: {info.IsOffline}");
sb.AppendLine($"--> Is marked as archived: {info.IsArchive}");
sb.AppendLine($"--> Is marked as HIDDEN: {info.IsHidden}");
sb.AppendLine($"--> Is a SYSTEM FILE: {info.IsSystem}");
sb.AppendLine($"--> Is encrypted by the operating system: {info.IsEncrypted}");
sb.AppendLine($"--> Is compressed by the operating system: {info.IsCompressed}");
sb.AppendLine($"--> Is a mount point: {info.IsMountPoint}");
sb.AppendLine($"--> Is temporary: {info.IsTemporary}");
sb.AppendLine($"--> Is a symbolic link: {info.IsSymbolicLink}");
sb.AppendLine($"--> Is a sparse file: {info.IsSparseFile}");
sb.AppendLine($"--> Is a reparse point: {info.IsReparsePoint}");
sb.AppendLine($"--> Is not content indexed by the operating system: {info.IsNotContentIndexed}");
sb.AppendLine($"--> Is a directory: {info.IsDirectory}");
sb.AppendLine($"--> Is a device: {info.IsDevice}");
sb.AppendLine($"--> Is a normal file: {info.IsNormal}");
sb.AppendLine($"--> File size: {info.FileSize} bytes");
is_readonly = info.IsReadOnly;
if (info.IsOffline || info.IsHidden || info.IsSystem || info.IsEncrypted || info.IsMountPoint || info.IsTemporary || info.IsSymbolicLink || info.IsSparseFile || info.IsReparsePoint || info.IsDirectory || info.IsDevice)
{
sb.AppendLine("");
sb.AppendLine("--> WARNING: this doesn't look like a very normal file.");
sb.AppendLine(" Check the attributes above to determine if they are as you expect.");
sb.AppendLine("");
looks_sane = false;
}
}
catch (Exception ex2)
{
sb.AppendLine($"--> FAILED to collect the file attributes: {ex2.ToStringAllExceptionDetails()}");
looks_sane = false;
}
// Check if we can open the file for basic I/O:
try
{
// read the entire file into a 1M buffer. Watch for errors.
byte[] buf = new byte[1024 * 1024];
const int count = 1024 * 1024;
long length_read = 0;
using (var stream = File.OpenRead(library_path))
{
int rc;
while (true)
{
rc = stream.Read(buf, 0, count);
if (rc > 0)
{
length_read += rc;
}
else if (rc == 0)
{
// EOF
break;
}
else
{
throw new Exception($"stream.Read produced a negative result: {rc}");
}
}
}
long file_size = File.GetSize(library_path);
if (file_size != length_read)
{
throw new Exception($"stream.Read was unable to read/scan the entire file:\n file size reported by the file system = {file_size} bytes, length scanned = {length_read} bytes");
}
sb.AppendLine($"--> Successfully scanned the entire file: length scanned = {length_read} bytes");
}
catch (Exception ex2)
{
sb.AppendLine($"--> FAILED to read/scan the file: {ex2.ToStringAllExceptionDetails()}");
looks_sane = false;
}
if (!is_readonly)
{
// check if we can open the file for WRITE access
try
{
using (var stream = File.OpenWrite(library_path))
{
sb.AppendLine($"--> Successfully opened the file for WRITE ACCESS");
}
}
catch (Exception ex2)
{
sb.AppendLine($"--> FAILED to open the file for WRITE ACCESS:);");
sb.AppendLine(ex2.ToStringAllExceptionDetails());
looks_sane = false;
}
}
if (corruption_list != null && corruption_list.Count > 0)
{
if (looks_sane)
{
sb.AppendLine("--> WARNING: while the RAW file scan and access checks may have PASSED,");
sb.AppendLine(" the Qiqqa inner systems certainly found stuff in the file to complain about:");
sb.AppendLine(" these data corruptions (a.k.a. DECODE FAILURES) have already been reported,");
sb.AppendLine(" but here they are once more in summarized form:");
}
else
{
sb.AppendLine("--> ERROR: while the RAW file scan and access checks may have FAILED,");
sb.AppendLine(" the Qiqqa inner systems also found stuff in the file to complain about");
sb.AppendLine(" -- which is VERY PROBABLY related to or caused by the findings reported above.");
sb.AppendLine("");
sb.AppendLine(" The data corruptions (a.k.a. DECODE FAILURES) have already been reported,");
sb.AppendLine(" but here they are once more in summarized form:");
}
sb.AppendLine("");
int index = 1;
foreach (var corruption in corruption_list)
{
sb.AppendLine($" #{index.ToString("03")}: {corruption.Message.Split('\n')[0]}");
index++;
}
sb.AppendLine($" --- {corruption_list.Count} reported data corruptions ---------------------------------------");
looks_sane = false;
}
if (sql_ex != null)
{
sb.AppendLine("");
sb.AppendLine(" As this report is about a SQLite database error, it MAY be useful to search");
sb.AppendLine(" the Internet for generic help and/or discussion of the reported SQLite error:");
sb.AppendLine("");
int errorcode = sql_ex.ErrorCode;
int basenum = errorcode & 0xFF;
int extended_shift = errorcode >> 8;
int herr = sql_ex.HResult;
sb.AppendLine(" ( ref: https://sqlite.org/rescode.html )");
sb.AppendLine(" ( ref: https://sqlite.org/c3ref/c_abort.html )");
sb.AppendLine($" SQLite Error Code: {basenum}");
if (extended_shift != 0)
{
sb.AppendLine(" ( ref: https://sqlite.org/c3ref/c_abort_rollback.html )");
sb.AppendLine($" SQLite Extended Error Code: ({basenum} | ({extended_shift} << 8)) = {errorcode}");
}
else
{
sb.AppendLine(" Reported error code is NOT a SQLite Extended Error Code.");
}
sb.AppendLine($" SQLite HResult: {herr.ToString("08:X")}");
sb.AppendLine("");
sb.AppendLine($" SQLite: the define constants (i.e. compile-time options): {SQLiteConnection.DefineConstants}");
sb.AppendLine($" SQLite: the underlying SQLite core library: {SQLiteConnection.SQLiteVersion}");
sb.AppendLine($" SQLite: SQLITE_SOURCE_ID: {SQLiteConnection.SQLiteSourceId}");
sb.AppendLine($" SQLite: the compile-time options: {SQLiteConnection.SQLiteCompileOptions}");
sb.AppendLine($" SQLite: the version of the interop SQLite assembly: {SQLiteConnection.InteropVersion}");
sb.AppendLine($" SQLite: the unique identifier for the source checkout used to build the interop assembly: {SQLiteConnection.InteropSourceId}");
sb.AppendLine($" SQLite: the compile-time options used to compile the SQLite interop assembly: {SQLiteConnection.InteropCompileOptions}");
sb.AppendLine($" SQLite: the version of the managed components: {SQLiteConnection.ProviderVersion}");
sb.AppendLine($" SQLite: the unique identifier for the source checkout used to build the managed components: {SQLiteConnection.ProviderSourceId}");
sb.AppendLine($" SQLite: the extra connection flags: {SQLiteConnection.SharedFlags}");
sb.AppendLine($" SQLite: the default connection flags: {SQLiteConnection.DefaultFlags}");
sb.AppendLine("");
sb.AppendLine(" (ref: https://sqlite.org/c3ref/extended_result_codes.html )");
sb.AppendLine(" SQLite Extended Error Reporting has been ENABLED: SetExtendedResultCodes(true)");
}
sb.AppendLine("---------");
if (looks_sane)
{
sb.AppendLine("");
sb.AppendLine("--> VERDICT OK(?): this DOES look like a very normal file.");
sb.AppendLine("");
sb.AppendLine(" HOWEVER, it may have incorrect a.k.a. 'corrupted' content, which made Qiqqa barf,");
sb.AppendLine(" or there's something else going on which this simply diagnosis routine");
sb.AppendLine(" is unable to unearth.");
sb.AppendLine("");
sb.AppendLine(" Please file a report at the Qiqqa issue tracker and include this logging");
sb.AppendLine(" for further inspection.");
}
else
{
sb.AppendLine("");
sb.AppendLine("--> VERDICT BAD(?): as far as this simple diagnostic routine can tell,");
sb.AppendLine(" this is NOT an 'okay' file.");
sb.AppendLine("");
if (is_readonly)
{
sb.AppendLine(" It MAY be marked READ-ONLY, which MAY be okay in your book, but is certainly");
sb.AppendLine(" unexpected here.");
sb.AppendLine("");
}
sb.AppendLine(" There's something going on which this simply diagnosis routine CANNOT diagnose further.");
sb.AppendLine("");
sb.AppendLine(" Please file a report at the Qiqqa issue tracker and include this logging");
sb.AppendLine(" for further inspection.");
}
sb.AppendLine("");
sb.AppendLine("==================== End of diagnostics report ============================================");
} while (false);
Logging.Error(sb.ToString());
}
private long SizeForID(string id)
{
return(File.GetSize(Path.Combine(_outputFolder, id)));
}