/// <summary>
/// Adds a file record to the database
/// </summary>
/// <param name="filename">The path to the file</param>
/// <param name="lastmodified">The time the file was modified</param>
/// <param name="blocksetID">The ID of the hashkey for the file</param>
/// <param name="metadataID">The ID for the metadata</param>
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
/// <param name="operationId">The operationId to use, or -1 to use the current operation</param>
public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, System.Data.IDbTransaction transaction)
{
var fileidobj = -1L;
PathEntryKeeper entry = null;
var entryFound = false;
if (m_pathLookup != null)
{
if (entryFound = (m_pathLookup.TryFind(filename, out entry) && entry != null))
{
var fid = entry.GetFilesetID(blocksetID, metadataID);
if (fid >= 0)
{
fileidobj = fid;
}
}
}
else
{
m_findfilesetCommand.Transaction = transaction;
m_findfilesetCommand.SetParameterValue(0, blocksetID);
m_findfilesetCommand.SetParameterValue(1, metadataID);
m_findfilesetCommand.SetParameterValue(2, filename);
fileidobj = m_findfilesetCommand.ExecuteScalarInt64();
}
if (fileidobj == -1)
{
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
m_insertfileCommand.Transaction = tr.Parent;
m_insertfileCommand.SetParameterValue(0, filename);
m_insertfileCommand.SetParameterValue(1, blocksetID);
m_insertfileCommand.SetParameterValue(2, metadataID);
fileidobj = m_insertfileCommand.ExecuteScalarInt64();
tr.Commit();
// We do not need to update this, because we will not ask for the same file twice
if (m_pathLookup != null)
{
if (!entryFound)
{
entry = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
entry.AddFilesetID(blocksetID, metadataID, fileidobj);
m_pathLookup.Insert(filename, entry);
}
else
{
entry.AddFilesetID(blocksetID, metadataID, fileidobj);
}
}
}
}
m_insertfileOperationCommand.Transaction = transaction;
m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
m_insertfileOperationCommand.SetParameterValue(1, fileidobj);
m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
m_insertfileOperationCommand.ExecuteNonQuery();
}
/// <summary>
/// Adds a file record to the database
/// </summary>
/// <param name="filename">The path to the file</param>
/// <param name="scantime">The time the file was scanned</param>
/// <param name="blocksetID">The ID of the hashkey for the file</param>
/// <param name="metadataID">The ID for the metadata</param>
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
/// <param name="operationId">The operationId to use, or -1 to use the current operation</param>
public void AddFile(string filename, DateTime scantime, long blocksetID, long metadataID, System.Data.IDbTransaction transaction)
{
object fileidobj = null;
PathEntryKeeper entry = null;
bool entryFound = false;
if (m_pathLookup != null)
{
if (entryFound = (m_pathLookup.TryFind(filename, out entry) && entry != null))
{
var fid = entry.GetFilesetID(blocksetID, metadataID);
if (fid >= 0)
{
fileidobj = fid;
}
}
}
else
{
m_findfilesetCommand.Transaction = transaction;
m_findfilesetCommand.SetParameterValue(0, blocksetID);
m_findfilesetCommand.SetParameterValue(1, metadataID);
m_findfilesetCommand.SetParameterValue(2, filename);
fileidobj = m_findfilesetCommand.ExecuteScalar();
}
if (fileidobj == null || fileidobj == DBNull.Value)
{
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
m_insertfileCommand.Transaction = tr.Parent;
m_insertfileCommand.SetParameterValue(0, filename);
m_insertfileCommand.SetParameterValue(1, blocksetID);
m_insertfileCommand.SetParameterValue(2, metadataID);
fileidobj = Convert.ToInt64(m_insertfileCommand.ExecuteScalar());
tr.Commit();
// We do not need to update this, because we will not ask for the same file twice
if (m_pathLookup != null)
{
if (!entryFound)
{
entry = new PathEntryKeeper(-1, DateTime.UtcNow);
entry.AddFilesetID(blocksetID, metadataID, Convert.ToInt64(fileidobj));
m_pathLookup.Insert(filename, entry);
}
else
{
entry.AddFilesetID(blocksetID, metadataID, Convert.ToInt64(fileidobj));
}
}
}
}
m_insertfileOperationCommand.Transaction = transaction;
m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
m_insertfileOperationCommand.SetParameterValue(1, fileidobj);
m_insertfileOperationCommand.SetParameterValue(2, NormalizeDateTimeToEpochSeconds(scantime));
m_insertfileOperationCommand.ExecuteNonQuery();
}
private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
{
var fileid = -1L;
var metadataid = AddMetadataset(metahash, metahashsize, transaction);
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
{
fileid = e.GetFilesetID(blocksetid, metadataid);
}
}
else
{
m_findFilesetCommand.Transaction = transaction;
m_findFilesetCommand.SetParameterValue(0, path);
m_findFilesetCommand.SetParameterValue(1, blocksetid);
m_findFilesetCommand.SetParameterValue(2, metadataid);
var r = m_findFilesetCommand.ExecuteScalar();
if (r != null && r != DBNull.Value)
{
fileid = Convert.ToInt64(r);
}
}
if (fileid < 0)
{
m_insertFileCommand.Transaction = transaction;
m_insertFileCommand.SetParameterValue(0, path);
m_insertFileCommand.SetParameterValue(1, blocksetid);
m_insertFileCommand.SetParameterValue(2, metadataid);
fileid = Convert.ToInt64(m_insertFileCommand.ExecuteScalar());
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
{
e.AddFilesetID(blocksetid, metadataid, fileid);
}
else
{
e = new PathEntryKeeper();
e.AddFilesetID(blocksetid, metadataid, fileid);
m_filesetLookup.Insert(path, e);
}
}
}
m_insertFilesetEntryCommand.Transaction = transaction;
m_insertFilesetEntryCommand.SetParameterValue(0, filesetid);
m_insertFilesetEntryCommand.SetParameterValue(1, fileid);
m_insertFilesetEntryCommand.SetParameterValue(2, NormalizeDateTimeToEpochSeconds(time));
m_insertFilesetEntryCommand.ExecuteNonQuery();
}
private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
{
var fileid = -1L;
var metadataid = AddMetadataset(metahash, metahashsize, transaction);
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
{
fileid = e.GetFilesetID(blocksetid, metadataid);
}
}
else
{
m_findFilesetCommand.Transaction = transaction;
m_findFilesetCommand.SetParameterValue(0, path);
m_findFilesetCommand.SetParameterValue(1, blocksetid);
m_findFilesetCommand.SetParameterValue(2, metadataid);
fileid = m_findFilesetCommand.ExecuteScalarInt64(-1);
}
if (fileid < 0)
{
m_insertFileCommand.Transaction = transaction;
m_insertFileCommand.SetParameterValue(0, path);
m_insertFileCommand.SetParameterValue(1, blocksetid);
m_insertFileCommand.SetParameterValue(2, metadataid);
fileid = m_insertFileCommand.ExecuteScalarInt64(-1);
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
{
e.AddFilesetID(blocksetid, metadataid, fileid);
}
else
{
e = new PathEntryKeeper();
e.AddFilesetID(blocksetid, metadataid, fileid);
m_filesetLookup.Insert(path, e);
}
}
}
m_insertFilesetEntryCommand.Transaction = transaction;
m_insertFilesetEntryCommand.SetParameterValue(0, filesetid);
m_insertFilesetEntryCommand.SetParameterValue(1, fileid);
m_insertFilesetEntryCommand.SetParameterValue(2, time.ToUniversalTime().Ticks);
m_insertFilesetEntryCommand.ExecuteNonQuery();
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
//Need a temporary table with path/lastmodified lookups
var scantableDefinition =
@"SELECT ""A1"".""ID"" AS ""FileID"", ""A1"".""Lastmodified"" AS ""Lastmodified"", ""A1"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"", ""A1"".""BlocksetID"" " +
@" FROM (SELECT ""File"".""ID"", ""File"".""BlocksetID"", ""File"".""MetadataID"", ""FilesetEntry"".""Lastmodified"", ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A1"" LEFT JOIN " +
@" (SELECT ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A2"" ON ""A1"".""Path"" = ""A2"".""Path"" AND ""A1"".""Timestamp"" < ""A2"".""Timestamp"" " +
@" , ""Blockset"" ""C"", ""Metadataset"" ""E"", ""Blockset"" ""F"" " +
@" WHERE ""A2"".""Path"" IS NULL " +
@" AND ""C"".""ID"" = ""A1"".""BlocksetID"" " +
@" AND ""A1"".""MetadataID"" = ""E"".""ID"" " +
@" AND ""F"".""ID"" = ""E"".""BlocksetID"" ";
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path lastmodified lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ({0}) WHERE ""BlocksetID"" >= 0 ", scantableDefinition)))
while (rd.Read())
{
var id = rd.GetInt64(0);
var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc);
var filesize = rd.GetInt64(2);
var path = rd.GetString(3);
var metahash = rd.GetString(4);
var metasize = rd.GetInt64(5);
m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize));
}
}
if (m_pathLookup != null)
{
try
{
using (new Logging.Timer("Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = rd.GetInt64(1);
var metadataid = rd.GetInt64(2);
var filesetid = rd.GetInt64(3);
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate file entries detected, run repair to fix it", ex);
}
}
m_missingBlockHashes = cmd.ExecuteScalarInt64(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Block"".""VolumeID"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
if (tc > 0)
{
throw new InvalidDataException("Detected blocks that are not reachable in the block table");
}
}
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.BlockHashLookupMemory > 0)
{
m_blockHashLookup = new HashLookupHelper <KeyValuePair <long, long> >((ulong)options.BlockHashLookupMemory);
}
if (options.FileHashLookupMemory > 0)
{
m_fileHashLookup = new HashLookupHelper <long>((ulong)options.FileHashLookupMemory);
}
if (options.MetadataHashMemory > 0)
{
m_metadataLookup = new HashLookupHelper <long>((ulong)options.MetadataHashMemory);
}
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
if (m_blockHashLookup != null)
{
using (new Logging.Timer("Build blockhash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""ID"", ""Block"".""Size"" FROM ""Block"" "))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var id = Convert.ToInt64(rd.GetValue(1));
var size = Convert.ToInt64(rd.GetValue(2));
m_blockHashLookup.Add(str, size, new KeyValuePair <long, long>(id, size));
}
}
if (m_fileHashLookup != null)
{
using (new Logging.Timer("Build filehash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""Length"", ""ID"" FROM ""BlockSet"""))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var size = Convert.ToInt64(rd.GetValue(1));
var id = Convert.ToInt64(rd.GetValue(2));
m_fileHashLookup.Add(str, size, id);
}
}
if (m_metadataLookup != null)
{
using (new Logging.Timer("Build metahash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT ""Metadataset"".""ID"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""Metadataset"", ""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "))
while (rd.Read())
{
var metadataid = Convert.ToInt64(rd.GetValue(0));
var hash = rd.GetValue(1).ToString();
var size = Convert.ToInt64(rd.GetValue(2));
m_metadataLookup.Add(hash, size, metadataid);
}
}
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path scantime lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Scantime"", ""Path"", ""Scantime"" FROM ""{0}"" WHERE ""BlocksetID"" >= 0 ", m_scantimelookupTablename)))
while (rd.Read())
{
var id = Convert.ToInt64(rd.GetValue(0));
var scantime = ParseFromEpochSeconds(Convert.ToInt64(rd.GetValue(1)));
var path = rd.GetValue(2).ToString();
m_pathLookup.Insert(path, new PathEntryKeeper(id, scantime));
}
}
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = Convert.ToInt64(rd.GetValue(1));
var metadataid = Convert.ToInt64(rd.GetValue(2));
var filesetid = Convert.ToInt64(rd.GetValue(3));
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, DateTime.UtcNow);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
m_missingBlockHashes = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Block"".""VolumeID"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()));
}
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
//Need a temporary table with path/lastmodified lookups
var scantableDefinition =
@"SELECT ""A1"".""ID"" AS ""FileID"", ""A1"".""Lastmodified"" AS ""Lastmodified"", ""A1"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"", ""A1"".""BlocksetID"" " +
@" FROM (SELECT ""File"".""ID"", ""File"".""BlocksetID"", ""File"".""MetadataID"", ""FilesetEntry"".""Lastmodified"", ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A1"" LEFT JOIN " +
@" (SELECT ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A2"" ON ""A1"".""Path"" = ""A2"".""Path"" AND ""A1"".""Timestamp"" < ""A2"".""Timestamp"" " +
@" , ""Blockset"" ""C"", ""Metadataset"" ""E"", ""Blockset"" ""F"" " +
@" WHERE ""A2"".""Path"" IS NULL " +
@" AND ""C"".""ID"" = ""A1"".""BlocksetID"" " +
@" AND ""A1"".""MetadataID"" = ""E"".""ID"" " +
@" AND ""F"".""ID"" = ""E"".""BlocksetID"" ";
if (m_pathLookup != null)
{
using (new Logging.Timer(LOGTAG, "BuildLastModified", "Build path lastmodified lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ({0}) WHERE ""BlocksetID"" >= 0 ", scantableDefinition)))
while (rd.Read())
{
var id = rd.GetInt64(0);
var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc);
var filesize = rd.GetInt64(2);
var path = rd.GetString(3);
var metahash = rd.GetString(4);
var metasize = rd.GetInt64(5);
m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize));
}
}
if (m_pathLookup != null)
{
try
{
using (new Logging.Timer(LOGTAG, "BuildPathTable", "Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = rd.GetInt64(1);
var metadataid = rd.GetInt64(2);
var filesetid = rd.GetInt64(3);
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate file entries detected, run repair to fix it", ex);
}
}
var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
if (tc > 0)
{
throw new InvalidDataException("Detected blocks that are not reachable in the block table");
}
}
if (options.UseBlockCache)
{
string failedhash = null;
try
{
var cache = new Dictionary <string, long>();
using (var cmd = m_connection.CreateCommand())
{
cmd.CommandText = @"SELECT ""Hash"", ""Size"" From ""Block""";
using (var rd = cmd.ExecuteReader())
while (rd.Read())
{
cache.Add(failedhash = rd.ConvertValueToString(0), rd.ConvertValueToInt64(1));
}
}
m_blockCache = cache;
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheFailure", ex, "Failed to create block cache, this could mean you have hash collisions in your table, the hash that failed is {0}. Error message: {1}.", failedhash, ex.Message);
Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheFailure", null, "Disabling block cache due to error");
}
}
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.BlockHashLookupMemory > 0)
{
m_blockHashLookup = new HashLookupHelper <KeyValuePair <long, long> >((ulong)options.BlockHashLookupMemory);
}
if (options.FileHashLookupMemory > 0)
{
m_fileHashLookup = new HashLookupHelper <long>((ulong)options.FileHashLookupMemory);
}
if (options.MetadataHashMemory > 0)
{
m_metadataLookup = new HashLookupHelper <long>((ulong)options.MetadataHashMemory);
}
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
//Need a temporary table with path/lastmodified lookups
var scantableDefinition = @"SELECT ""B"".""ID"" AS ""FileID"", ""D"".""Lastmodified"" AS ""Lastmodified"", ""B"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"" FROM (SELECT ""FilesetEntry"".""FileID"" AS ""FileID"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"", MAX(""Fileset"".""Timestamp"") AS ""MostRecent"" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" GROUP BY ""File"".""Path"") A, ""File"" B, ""Blockset"" C, ""FilesetEntry"" D, ""Metadataset"" E, ""Blockset"" F WHERE ""B"".""ID"" = ""A"".""FileID"" AND ""C"".""ID"" = ""B"".""BlocksetID"" AND ""D"".""FileID"" = ""B"".""ID"" AND ""D"".""FilesetID"" = ""A"".""FilesetID"" AND ""B"".""MetadataID"" = ""E"".""ID"" AND ""F"".""ID"" = ""E"".""BlocksetID""";
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + scantableDefinition, m_lastmodifiedLookupTablename));
cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}Index"" ON ""{0}"" (""Path"", ""Lastmodified"", ""Length"", ""Metahash"", ""Metasize"", ""FileID"") ", m_lastmodifiedLookupTablename));
if (m_blockHashLookup != null)
{
try
{
using (new Logging.Timer("Build blockhash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""ID"", ""Block"".""Size"" FROM ""Block"" "))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var id = rd.GetInt64(1);
var size = rd.GetInt64(2);
m_blockHashLookup.Add(str, size, new KeyValuePair <long, long>(id, size));
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate blockhashes detected, either repair the database or rebuild it", ex);
}
}
if (m_fileHashLookup != null)
{
try
{
using (new Logging.Timer("Build filehash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""Length"", ""ID"" FROM ""BlockSet"""))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var size = rd.GetInt64(1);
var id = rd.GetInt64(2);
m_fileHashLookup.Add(str, size, id);
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate filehashes detected, either repair the database or rebuild it", ex);
}
}
if (m_metadataLookup != null)
{
try
{
using (new Logging.Timer("Build metahash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT ""Metadataset"".""ID"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""Metadataset"", ""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "))
while (rd.Read())
{
var metadataid = rd.GetInt64(0);
var hash = rd.GetValue(1).ToString();
var size = rd.GetInt64(2);;
m_metadataLookup.Add(hash, size, metadataid);
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate metadatahash detected, run repair to fix it", ex);
}
}
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path lastmodified lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ""{0}"" WHERE ""BlocksetID"" >= 0 ", m_lastmodifiedLookupTablename)))
while (rd.Read())
{
var id = rd.GetInt64(0);
var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc);
var filesize = rd.GetInt64(2);
var path = rd.GetString(3);
var metahash = rd.GetString(4);
var metasize = rd.GetInt64(5);
m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize));
}
}
if (m_pathLookup != null)
{
try
{
using (new Logging.Timer("Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = rd.GetInt64(1);
var metadataid = rd.GetInt64(2);
var filesetid = rd.GetInt64(3);
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate file entries detected, run repair to fix it", ex);
}
}
m_missingBlockHashes = cmd.ExecuteScalarInt64(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Block"".""VolumeID"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
if (tc > 0)
{
throw new InvalidDataException("Detected blocks that are not reachable in the block table");
}
}
}
