public void PutBlob(string filename, byte[] data)
{
// Guard
if (String.IsNullOrEmpty(filename))
{
throw new Exception("Can't store in LibraryDB with null filename.");
}
// Calculate the MD5 of this blobbiiiieeeeee
string md5 = StreamMD5.FromBytes(data);
using (var connection = GetConnection())
{
connection.Open();
using (var transaction = connection.BeginTransaction())
{
using (var command = new SQLiteCommand("DELETE FROM LibraryItem WHERE filename=@filename", connection, transaction))
{
command.Parameters.AddWithValue("@filename", filename);
command.ExecuteNonQuery();
}
using (var command = new SQLiteCommand("INSERT INTO LibraryItem(filename, last_updated_by, md5, data) VALUES(@filename, @last_updated_by, @md5, @data)", connection, transaction))
{
command.Parameters.AddWithValue("@filename", filename);
command.Parameters.AddWithValue("@last_updated_by", Environment.UserName);
command.Parameters.AddWithValue("@md5", md5);
command.Parameters.AddWithValue("@data", data);
command.ExecuteNonQuery();
}
transaction.Commit();
}
}
}
public string ScrapeURLToFile(string url, bool force_download = false, Dictionary <string, string> additional_headers = null)
{
string cache_key = StreamMD5.FromText(url);
string directory = Path.GetFullPath(Path.Combine(base_directory, cache_key.Substring(0, 2)));
string filename = Path.GetFullPath(Path.Combine(directory, cache_key));
if (!File.Exists(filename) || force_download)
{
Utilities.Files.DirectoryTools.CreateDirectory(directory);
// Crude throttle
if (true)
{
while (DateTime.UtcNow.Subtract(last_scrape_time).TotalMilliseconds < throttle_ms)
{
Thread.Sleep(50);
}
last_scrape_time = DateTime.UtcNow;
}
Logging.Info("Downloading from {0}", url);
using (WebClient client = new WebClient())
{
if (null != additional_headers)
{
foreach (var pair in additional_headers)
{
client.Headers.Add(pair.Key, pair.Value);
}
}
if (!String.IsNullOrEmpty(userAgent))
{
client.Headers.Add("User-agent", userAgent);
}
string temp_filename = filename + ".tmp";
try
{
client.DownloadFile(url, temp_filename);
}
catch (WebException ex)
{
File.WriteAllText(temp_filename, ex.ToString());
}
File.Delete(filename);
File.Move(temp_filename, filename);
}
string filename_manifest = Path.GetFullPath(Path.Combine(base_directory, @"manifest.txt"));
string manifest_line = String.Format("{0}\t{1}", cache_key, url);
using (StreamWriter sw = File.AppendText(filename_manifest))
{
sw.WriteLine(manifest_line);
}
}
return(filename);
}
internal void PutBlob(SQLiteConnection connection, SQLiteTransaction transaction, string fingerprint, string extension, byte[] data)
{
// Calculate the MD5 of this blobbiiiieeeeee
string md5 = StreamMD5.FromBytes(data);
using (var command = new SQLiteCommand("INSERT INTO LibraryItem(fingerprint, extension, md5, data) VALUES(@fingerprint, @extension, @md5, @data)", connection, transaction))
{
command.Parameters.AddWithValue("@fingerprint", fingerprint);
command.Parameters.AddWithValue("@extension", extension);
command.Parameters.AddWithValue("@md5", md5);
command.Parameters.AddWithValue("@data", data);
command.ExecuteNonQuery();
}
}
public void PutBlob(string fingerprint, string extension, byte[] data)
{
// Guard
if (String.IsNullOrEmpty(fingerprint))
{
throw new Exception("Can't store in LibraryDB with null fingerprint.");
}
if (String.IsNullOrEmpty(extension))
{
throw new Exception("Can't store in LibraryDB with null extension.");
}
// Calculate the MD5 of this blobbiiiieeeeee
string md5 = StreamMD5.FromBytes(data);
using (var connection = GetConnection())
{
connection.Open();
bool managed_update = false;
using (var command = new SQLiteCommand("UPDATE LibraryItem SET MD5=@md5, DATA=@data WHERE fingerprint=@fingerprint AND extension=@extension", connection))
{
command.Parameters.AddWithValue("@md5", md5);
command.Parameters.AddWithValue("@data", data);
command.Parameters.AddWithValue("@fingerprint", fingerprint);
command.Parameters.AddWithValue("@extension", extension);
int num_rows_updated = command.ExecuteNonQuery();
if (1 == num_rows_updated)
{
managed_update = true;
}
}
if (!managed_update)
{
using (var command = new SQLiteCommand("INSERT INTO LibraryItem(fingerprint, extension, md5, data) VALUES(@fingerprint, @extension, @md5, @data)", connection))
{
command.Parameters.AddWithValue("@fingerprint", fingerprint);
command.Parameters.AddWithValue("@extension", extension);
command.Parameters.AddWithValue("@md5", md5);
command.Parameters.AddWithValue("@data", data);
command.ExecuteNonQuery();
}
}
}
}
private string MkLegalSizedPath(string basename, string typeIdStr)
{
const int PATH_MAX = 240; // must be less than 255 / 260 - see also https://kb.acronis.com/content/39790
string root = Path.GetDirectoryName(basename);
string name = Path.GetFileName(basename);
string dataname = Path.GetFileNameWithoutExtension(DataFile);
string ext = SubStr(Path.GetExtension(DataFile), 1).Trim(); // produce the extension without leading dot
if (ext.StartsWith("bib"))
{
ext = SubStr(ext, 3).Trim();
}
if (ext.Length > 0)
{
ext = "." + ext;
}
// UNC long filename/path support by forcing this to be a UNC path:
string filenamebase = $"{dataname}.{name}{ext}{ExtensionWithDot}";
// first make the full path without the approved/received, so that that bit doesn't make a difference
// in the length check and subsequent decision to produce a shorthand filename path or not:
// It's not always needed, but do the different shorthand conversions anyway and pick the longest fitting one:
string short_tn = SanitizeFilename(CamelCaseShorthand(name));
string short_dn = SanitizeFilename(SubStr(dataname, 0, 10) + CamelCaseShorthand(dataname));
string hash = StreamMD5.FromText(filenamebase).ToUpper();
string short_hash = SubStr(hash, 0, Math.Max(6, 11 - short_tn.Length));
// this variant will fit in the length criterium, guaranteed:
string alt_filepath0 = Path.GetFullPath(Path.Combine(root, $"{short_dn}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}"));
string filepath = alt_filepath0;
// next, we construct the longer variants to check if they fit.
//
// DO NOTE that we create a path without typeIdStr part first, because we want both received and approved files to be based
// on the *same* alt selection decision!
string picked_alt_filepath = Path.GetFullPath(Path.Combine(root, $"{short_dn}.{short_hash}_{short_tn}{ext}.APPROVEDXYZ{ExtensionWithDot}"));
name = SanitizeFilename(name);
dataname = SanitizeFilename(dataname);
string alt_filepath1 = Path.GetFullPath(Path.Combine(root, $"{short_dn}_{short_hash}.{name}{ext}.APPROVEDXYZ{ExtensionWithDot}"));
if (alt_filepath1.Length < PATH_MAX)
{
filepath = Path.GetFullPath(Path.Combine(root, $"{short_dn}_{short_hash}.{name}{ext}{typeIdStr}{ExtensionWithDot}"));
picked_alt_filepath = alt_filepath1;
}
// second alternative: only pick this one if it fits and produces a longer name:
string alt_filepath2 = Path.GetFullPath(Path.Combine(root, $"{dataname}.{short_hash}_{short_tn}{ext}.APPROVEDXYZ{ExtensionWithDot}"));
if (alt_filepath2.Length < PATH_MAX && alt_filepath2.Length > picked_alt_filepath.Length)
{
filepath = Path.GetFullPath(Path.Combine(root, $"{dataname}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}"));
picked_alt_filepath = alt_filepath2;
}
else
{
// third alt: the 'optimally trimmed' test name used as part of the filename:
int trim_length = PATH_MAX - alt_filepath0.Length + 10 - 1;
string short_dn2 = SanitizeFilename(SubStr(dataname, 0, trim_length) + CamelCaseShorthand(dataname));
string alt_filepath3 = Path.GetFullPath(Path.Combine(root, $"{short_dn2}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}"));
if (alt_filepath3.Length < PATH_MAX && alt_filepath3.Length > picked_alt_filepath.Length)
{
filepath = Path.GetFullPath(Path.Combine(root, $"{short_dn2}.{short_hash}_{short_tn}{ext}{typeIdStr}{ExtensionWithDot}"));
picked_alt_filepath = alt_filepath3;
}
}
// fourth alt: the full, unadulterated path; if it fits in the length criterium, take it anyway
string alt_filepath4 = Path.GetFullPath(Path.Combine(root, $"{dataname}.{name}{ext}.APPROVEDXYZ{ExtensionWithDot}"));
if (alt_filepath4.Length < PATH_MAX)
{
// UNC long filename/path support by forcing this to be a UNC path:
filepath = Path.GetFullPath(Path.Combine(root, $"{dataname}.{name}{ext}{typeIdStr}{ExtensionWithDot}"));
picked_alt_filepath = alt_filepath4;
}
return(filepath);
}
public void PutBlob(string fingerprint, string extension, byte[] data)
{
// Guard
if (String.IsNullOrWhiteSpace(fingerprint))
{
throw new Exception("Can't store in LibraryDB with null fingerprint.");
}
if (String.IsNullOrWhiteSpace(extension))
{
throw new Exception("Can't store in LibraryDB with null extension.");
}
// Calculate the MD5 of this blobbiiiieeeeee
string md5 = StreamMD5.FromBytes(data);
try
{
lock (DBAccessLock.db_access_lock)
{
using (var connection = GetConnection())
{
connection.Open();
using (var transaction = connection.BeginTransaction())
{
bool managed_update = false;
using (var command = new SQLiteCommand("UPDATE LibraryItem SET MD5=@md5, DATA=@data WHERE fingerprint=@fingerprint AND extension=@extension", connection, transaction))
{
command.Parameters.AddWithValue("@md5", md5);
command.Parameters.AddWithValue("@data", data);
command.Parameters.AddWithValue("@fingerprint", fingerprint);
command.Parameters.AddWithValue("@extension", extension);
int num_rows_updated = command.ExecuteNonQuery();
if (1 == num_rows_updated)
{
managed_update = true;
}
}
if (!managed_update)
{
using (var command = new SQLiteCommand("INSERT INTO LibraryItem(fingerprint, extension, md5, data) VALUES(@fingerprint, @extension, @md5, @data)", connection, transaction))
{
command.Parameters.AddWithValue("@fingerprint", fingerprint);
command.Parameters.AddWithValue("@extension", extension);
command.Parameters.AddWithValue("@md5", md5);
command.Parameters.AddWithValue("@data", data);
command.ExecuteNonQuery();
}
}
transaction.Commit();
}
connection.Close();
}
//
// see SO link above at the `DBAccessLock.db_access_lock` declaration.
//
// We keep this *inside* the critical section so that we know we'll be the only active SQLite
// action which just transpired.
// *This* is also the reason why I went with a *global* lock (singleton) for *all* databases,
// even while *theoretically* this is *wrong* or rather: *unnecessary* as the databases
// i.e. Qiqqa Libraries shouldn't bite one another. I, however, need to ensure that the
// added `System.Data.SQLite.SQLiteConnection.ClearAllPools();` statements don't foul up
// matters in library B while lib A I/O is getting cleaned up.
//
// In short: Yuck. + Cave canem.
//
SQLiteConnection.ClearAllPools();
}
}
catch (Exception ex)
{
Logging.Error(ex, "LibraryDB::PutBLOB: Database I/O failure for DB '{0}'.", library_path);
LibraryDB.FurtherDiagnoseDBProblem(ex, null, library_path);
throw;
}
}
private static int DoDownloads(Library library, bool restricted_metadata_sync, Dictionary <string, string> historical_sync_file, SynchronisationAction synchronisation_action)
{
int download_count = 0;
StatusManager.Instance.ClearCancelled(StatusCodes.SYNC_META(library));
foreach (SynchronisationState ss in synchronisation_action.states_to_download)
{
StatusManager.Instance.UpdateStatus(StatusCodes.SYNC_META(library), String.Format("Downloading metadata from your Web/Intranet Library ({0} to go)", synchronisation_action.states_to_download.Count - download_count), download_count, synchronisation_action.states_to_download.Count, true);
++download_count;
// Has the user cancelled?
if (StatusManager.Instance.IsCancelled(StatusCodes.SYNC_META(library)))
{
Logging.Info("User has cancelled their metadata download");
break;
}
try
{
Logging.Info("+Downloading {0}", ss.filename);
StoredUserFile stored_user_file = null;
// --- TODO: Replace this with a pretty interface class ------------------------------------------------
if (false)
{
}
else if (library.WebLibraryDetail.IsIntranetLibrary)
{
stored_user_file = SynchronisationExecutor_Intranet.DoDownload(library, ss);
}
else
{
throw new Exception(String.Format("Did not understand how to download for library {0}", library.WebLibraryDetail.Title));
}
// -----------------------------------------------------------------------------------------------------
Logging.Info("-Downloading {0}", ss.filename);
{
// Check that the MD5s match, or we have had some issue in the download
Logging.Info("Checking content");
string md5_metadata = StreamMD5.FromBytes(stored_user_file.Content);
string header_etag = stored_user_file.Md5;
string header_etag_nik = header_etag;
if (null != header_etag && !String.IsNullOrEmpty(header_etag) && 0 != String.Compare(md5_metadata, header_etag, true) && 0 != String.Compare(md5_metadata, header_etag_nik, true))
{
throw new Exception(String.Format("Local and remote MD5s do not match. local={0} remote={1} remote_nik={2}", md5_metadata, header_etag, header_etag_nik));
}
Logging.Info("Copying content");
library.LibraryDB.PutBlob(ss.fingerprint, ss.extension, stored_user_file.Content);
// Remember this MD5 for our sync clash detection
Logging.Info("Remembering md5");
historical_sync_file[ss.filename] = md5_metadata;
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem downloading one of your sync files");
}
}
StatusManager.Instance.UpdateStatus(StatusCodes.SYNC_META(library), String.Format("Downloaded {0} metadata from your Web/Intranet Library", download_count));
return(download_count);
}
public void PutBlob(string filename, byte[] data)
{
// Guard
if (String.IsNullOrWhiteSpace(filename))
{
throw new Exception("Can't store in LibraryDB with null filename.");
}
// Calculate the MD5 of this blobbiiiieeeeee
string md5 = StreamMD5.FromBytes(data);
lock (DBAccessLock.db_access_lock)
{
using (var connection = GetConnection())
{
connection.Open();
using (var transaction = connection.BeginTransaction())
{
bool managed_update = false;
using (var command = new SQLiteCommand("UPDATE LibraryItem SET MD5=@md5, DATA=@data, LAST_UPDATED_BY=@last_updated_by WHERE filename=@filename", connection, transaction))
{
command.Parameters.AddWithValue("@filename", filename);
command.Parameters.AddWithValue("@last_updated_by", Environment.UserName);
command.Parameters.AddWithValue("@md5", md5);
command.Parameters.AddWithValue("@data", data);
int num_rows_updated = command.ExecuteNonQuery();
if (1 == num_rows_updated)
{
managed_update = true;
}
}
if (!managed_update)
{
using (var command = new SQLiteCommand("INSERT INTO LibraryItem(filename, last_updated_by, md5, data) VALUES(@filename, @last_updated_by, @md5, @data)", connection, transaction))
{
command.Parameters.AddWithValue("@filename", filename);
command.Parameters.AddWithValue("@last_updated_by", Environment.UserName);
command.Parameters.AddWithValue("@md5", md5);
command.Parameters.AddWithValue("@data", data);
command.ExecuteNonQuery();
}
}
transaction.Commit();
}
connection.Close();
}
//
// see SO link in ../LibraryDB.cs at the `DBAccessLock.db_access_lock` declaration.
//
// We keep this *inside* the critical section so that we know we'll be the only active SQLite
// action which just transpired.
// *This* is also the reason why I went with a *global* lock (singeton) for *all* databases,
// even while *theoretically* this is *wrong* or rather: *unneccessary* as the databases
// i.e. Qiqqa Libraries shouldn't bite one another. I, however, need to ensure that the
// added `System.Data.SQLite.SQLiteConnection.ClearAllPools();` statements don't foul up
// matters in another library while lib A I/O is getting cleaned up.
//
// In short: Yuck. + Cave canem.
//
SQLiteConnection.ClearAllPools();
}
}