internal static void AlertUserAboutProblematicImports()
{
bool do_view = MessageBoxes.AskErrorQuestion("There were problems with some of the documents you were trying to add to Qiqqa. Do you want to see the problem details?", true);
// do NOT spend a long time inside the lock!
// hence we null the report file reference so that other threads can
// create another report file and continue work while the user takes
// a slow look at the old one...
//
// In short: take `Process.Start(...)` *outside* the lock!
string report_filename = null;
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (problematic_import_documents_lock)
{
l1_clk.LockPerfTimerStop();
if (do_view)
{
report_filename = problematic_import_documents_filename;
}
else
{
File.Delete(problematic_import_documents_filename);
}
// reset:
problematic_import_documents_filename = null;
problematic_import_documents_alert_showing = 0;
}
if (null != report_filename)
{
Process.Start(report_filename);
}
}
public List <LibraryItem> GetLibraryItems(string extension, List <string> fingerprints = null, int MaxRecordCount = 0)
{
List <LibraryItem> results = new List <LibraryItem>();
List <Exception> database_corruption = new List <Exception>();
try
{
lock (DBAccessLock.db_access_lock)
{
using (var connection = GetConnection())
{
connection.Open();
string command_string = "SELECT fingerprint, extension, md5, data FROM LibraryItem WHERE 1=1 ";
command_string += turnArgumentSetIntoQueryPart("fingerprint", fingerprints);
command_string += turnArgumentIntoQueryPart("extension", extension);
if (MaxRecordCount > 0)
{
// http://www.sqlitetutorial.net/sqlite-limit/
command_string += " LIMIT @maxnum";
}
using (var command = new SQLiteCommand(command_string, connection))
{
//turnArgumentIntoQueryParameter(command, "fingerprint", fingerprint);
turnArgumentIntoQueryParameter(command, "extension", extension);
if (MaxRecordCount > 0)
{
command.Parameters.AddWithValue("@maxnum", MaxRecordCount);
}
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
LibraryItem result = new LibraryItem();
int field_count = 0;
// Read the record in 2-3 gangs, as there's some DBs out there which have the BLOB as NOT A BLOB but as a STRING type instead:
// this is probably caused by manual editing (using SQLite CLI or other means) of the BLOB record.
// gang 1: load the field count and header fields: these almost never fail.
try
{
field_count = reader.FieldCount;
result.fingerprint = reader.GetString(0);
result.extension = reader.GetString(1);
result.md5 = reader.GetString(2);
}
catch (Exception ex)
{
string msg = String.Format("LibraryDB::GetLibraryItems: Database record #{4} gang 1 decode failure for DB '{0}': fingerprint={1}, ext={2}, md5={3}, field_count={5}.",
library_path,
String.IsNullOrEmpty(result.fingerprint) ? "???" : result.fingerprint,
String.IsNullOrEmpty(result.extension) ? "???" : result.extension,
String.IsNullOrEmpty(result.md5) ? "???" : result.md5,
reader.StepCount, // ~= results.Count + database_corruption.Count
field_count
);
Logging.Error(ex, "{0}", msg);
Exception ex2 = new Exception(msg, ex);
database_corruption.Add(ex2);
// it's no use to try to decode the rest of the DB record: it is lost to us
continue;
}
if (true)
{
Exception ex2 = null;
long total_bytes = 0;
// gang 2: get the BLOB
try
{
total_bytes = reader.GetBytes(3, 0, null, 0, 0);
result.data = new byte[total_bytes];
long total_bytes2 = reader.GetBytes(3, 0, result.data, 0, (int)total_bytes);
if (total_bytes != total_bytes2)
{
throw new Exception("Error reading blob - blob size different on each occasion.");
}
results.Add(result);
continue;
}
catch (Exception ex)
{
string msg = String.Format("LibraryDB::GetLibraryItems: Database record #{4} BLOB decode failure for DB '{0}': fingerprint={1}, ext={2}, md5={3}, BLOB length={5}.",
library_path,
String.IsNullOrEmpty(result.fingerprint) ? "???" : result.fingerprint,
String.IsNullOrEmpty(result.extension) ? "???" : result.extension,
String.IsNullOrEmpty(result.md5) ? "???" : result.md5,
reader.StepCount, // ~= results.Count + database_corruption.Count
total_bytes
);
ex2 = new Exception(msg, ex);
// gang 3: get at the BLOB-née-STRING in an indirect way
object[] fields = new object[5];
try
{
reader.GetValues(fields);
byte[] arr = fields[3] as byte[];
if (arr != null)
{
string blob = Encoding.UTF8.GetString(arr, 0, arr.Length);
result.data = new byte[arr.Length];
Array.Copy(arr, result.data, arr.Length);
results.Add(result);
Logging.Warn("LibraryDB::GetLibraryItems: Database record #{0} BLOB field is instead decoded as UTF8 STRING, following this RESOLVED ERROR: {1}\n Decoded STRING content:\n{2}",
reader.StepCount, // ~= results.Count + database_corruption.Count
ex2.ToStringAllExceptionDetails(),
blob);
continue;
}
else
{
throw new Exception("Cannot extract BLOB field.");
}
}
catch (Exception ex3)
{
Logging.Error(ex2);
Logging.Error(ex3);
database_corruption.Add(ex2);
}
}
}
}
reader.Close();
}
}
connection.Close();
}
//
// see SO link above at the `DBAccessLock.db_access_lock` declaration.
//
// We keep this *inside* the critical section so that we know we'll be the only active SQLite
// action which just transpired.
// *This* is also the reason why I went with a *global* lock (singleton) for *all* databases,
// even while *theoretically* this is *wrong* or rather: *unnecessary* as the databases
// i.e. Qiqqa Libraries shouldn't bite one another. I, however, need to ensure that the
// added `System.Data.SQLite.SQLiteConnection.ClearAllPools();` statements don't foul up
// matters in library B while lib A I/O is getting cleaned up.
//
// In short: Yuck. + Cave canem.
//
SQLiteConnection.ClearAllPools();
}
}
catch (Exception ex)
{
Logging.Error(ex, "LibraryDB::GetLibraryItems: Database I/O failure for DB '{0}'.", library_path);
LibraryDB.FurtherDiagnoseDBProblem(ex, database_corruption, library_path);
throw;
}
if (database_corruption.Count > 0)
{
// report database corruption: the user may want to recover from this ASAP!
if (MessageBoxes.AskErrorQuestion(true, "Library '{0}' has some data corruption. Do you want to abort the application to attempt recovery using external tools, e.g. a data restore from backup?\n\nWhen you answer NO, we will continue with what we could recover so far instead.\n\n\nConsult the Qiqqa logfiles to see the individual corruptions reported.",
library_path))
{
Logging.Warn("User chose to abort the application on database corruption report");
Environment.Exit(3);
}
}
return(results);
}
public static PDFDocument AddNewPDFDocumentsToLibraryWithMetadata_SYNCHRONOUS(Library library, bool suppress_notifications, bool suppress_signal_that_docs_have_changed, FilenameWithMetadataImport[] filename_with_metadata_imports)
{
// Notify if there is just a single doc
suppress_notifications = suppress_notifications || (filename_with_metadata_imports.Length > 1);
StatusManager.Instance.ClearCancelled("BulkLibraryDocument");
PDFDocument last_added_pdf_document = null;
string problematic_import_documents_filename = null;
int successful_additions = 0;
for (int i = 0; i < filename_with_metadata_imports.Length; ++i)
{
if (StatusManager.Instance.IsCancelled("BulkLibraryDocument"))
{
Logging.Warn("User chose to stop bulk adding documents to the library");
break;
}
StatusManager.Instance.UpdateStatus("BulkLibraryDocument", String.Format("Adding document {0} of {1} to your library", i, filename_with_metadata_imports.Length), i, filename_with_metadata_imports.Length, true);
FilenameWithMetadataImport filename_with_metadata_import = filename_with_metadata_imports[i];
try
{
string filename = filename_with_metadata_import.filename;
string bibtex = filename_with_metadata_import.bibtex;
// Although the outside world may allow us to be signalling, we will not do it unless we are the n-100th doc or the last doc
bool local_suppress_signal_that_docs_have_changed = suppress_signal_that_docs_have_changed;
if (!local_suppress_signal_that_docs_have_changed)
{
if ((i != filename_with_metadata_imports.Length - 1) && (0 != i % 100))
{
local_suppress_signal_that_docs_have_changed = true;
}
}
PDFDocument pdf_document = library.AddNewDocumentToLibrary_SYNCHRONOUS(filename, filename, bibtex, filename_with_metadata_import.tags, filename_with_metadata_import.notes, suppress_notifications, local_suppress_signal_that_docs_have_changed);
if (null != pdf_document)
{
++successful_additions;
}
last_added_pdf_document = pdf_document;
}
catch (Exception ex)
{
Logging.Warn(ex, "There was a problem adding a document to the library:\n{0}", filename_with_metadata_import);
if (null == problematic_import_documents_filename)
{
problematic_import_documents_filename = TempFile.GenerateTempFilename("txt");
File.AppendAllText(
problematic_import_documents_filename,
"The following files caused problems while being imported into Qiqqa:\r\n\r\n"
);
}
File.AppendAllText(
problematic_import_documents_filename,
String.Format(
"----------\r\n{0}\r\n{1}\r\n----------\r\n"
, ex.Message
, filename_with_metadata_import
)
);
}
}
StatusManager.Instance.UpdateStatus("BulkLibraryDocument", String.Format("Added {0} of {1} document(s) to your library", successful_additions, filename_with_metadata_imports.Length));
// If there have been some import problems, report them to the user
if (null != problematic_import_documents_filename)
{
if (MessageBoxes.AskErrorQuestion("There were problems with some of the documents you were trying to add to Qiqqa. Do you want to see the problem details?", true))
{
Process.Start(problematic_import_documents_filename);
}
else
{
File.Delete(problematic_import_documents_filename);
}
}
return(last_added_pdf_document);
}
public List <IntranetLibraryItem> GetIntranetLibraryItemsSummary()
{
List <IntranetLibraryItem> results = new List <IntranetLibraryItem>();
List <Exception> database_corruption = new List <Exception>();
try
{
lock (DBAccessLock.db_access_lock)
{
using (var connection = GetConnection())
{
connection.Open();
string command_string = "SELECT filename, md5 FROM LibraryItem WHERE 1=1 ";
using (var command = new SQLiteCommand(command_string, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
IntranetLibraryItem result = new IntranetLibraryItem();
results.Add(result);
try
{
result.filename = reader.GetString(0);
result.md5 = reader.GetString(1);
}
catch (Exception ex)
{
string msg = String.Format("IntranetLibraryDB::GetIntranetLibraryItemsSummary: Database record #{3} decode failure for DB '{0}': filename_id={1}, md5={2}.",
library_path,
String.IsNullOrEmpty(result.filename) ? "???" : result.filename,
String.IsNullOrEmpty(result.md5) ? "???" : result.md5,
reader.StepCount // ~= results.Count + database_corruption.Count
);
Logging.Error(ex, "{0}", msg);
Exception ex2 = new Exception(msg, ex);
database_corruption.Add(ex2);
}
}
reader.Close();
}
}
connection.Close();
}
//
// see SO link above at the `DBAccessLock.db_access_lock` declaration.
//
// We keep this *inside* the critical section so that we know we'll be the only active SQLite
// action which just transpired.
// *This* is also the reason why I went with a *global* lock (singleton) for *all* databases,
// even while *theoretically* this is *wrong* or rather: *unnecessary* as the databases
// i.e. Qiqqa Libraries shouldn't bite one another. I, however, need to ensure that the
// added `System.Data.SQLite.SQLiteConnection.ClearAllPools();` statements don't foul up
// matters in library B while lib A I/O is getting cleaned up.
//
// In short: Yuck. + Cave canem.
//
SQLiteConnection.ClearAllPools();
}
}
catch (Exception ex)
{
Logging.Error(ex, "IntranetLibraryDB::GetLibraryItemsSummary: Database I/O failure for DB '{0}'.", library_path);
LibraryDB.FurtherDiagnoseDBProblem(ex, database_corruption, library_path);
throw;
}
if (database_corruption.Count > 0)
{
// report database corruption: the user may want to recover from this ASAP!
if (MessageBoxes.AskErrorQuestion(true, "INTRANET Library (Sync Point) '{0}' has some data corruption. Do you want to abort the application to attempt recovery using external tools, e.g. a data restore from backup?\n\nWhen you answer NO, we will continue with what we could recover so far instead.\n\n\nConsult the Qiqqa logfiles to see the individual corruptions reported.",
library_path))
{
Logging.Warn("User chose to abort the application on database corruption report");
Environment.Exit(3);
}
}
return(results);
}