public override void Dispose()
{
m_fileHashLookup = null;
m_metadataLookup = null;
m_blockHashLookup = null;
m_pathLookup = null;
base.Dispose();
}
public override void Dispose()
{
if (m_scantimelookupTablename != null)
{
try
{
using (var cmd = m_connection.CreateCommand())
cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_scantimelookupTablename));
}
catch { }
finally
{
m_scantimelookupTablename = null;
}
}
m_fileHashLookup = null;
m_metadataLookup = null;
m_blockHashLookup = null;
m_pathLookup = null;
base.Dispose();
}
public override void Dispose()
{
m_pathLookup = null;
base.Dispose();
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
//Need a temporary table with path/lastmodified lookups
var scantableDefinition =
@"SELECT ""A1"".""ID"" AS ""FileID"", ""A1"".""Lastmodified"" AS ""Lastmodified"", ""A1"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"", ""A1"".""BlocksetID"" " +
@" FROM (SELECT ""File"".""ID"", ""File"".""BlocksetID"", ""File"".""MetadataID"", ""FilesetEntry"".""Lastmodified"", ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A1"" LEFT JOIN " +
@" (SELECT ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A2"" ON ""A1"".""Path"" = ""A2"".""Path"" AND ""A1"".""Timestamp"" < ""A2"".""Timestamp"" " +
@" , ""Blockset"" ""C"", ""Metadataset"" ""E"", ""Blockset"" ""F"" " +
@" WHERE ""A2"".""Path"" IS NULL " +
@" AND ""C"".""ID"" = ""A1"".""BlocksetID"" " +
@" AND ""A1"".""MetadataID"" = ""E"".""ID"" " +
@" AND ""F"".""ID"" = ""E"".""BlocksetID"" ";
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path lastmodified lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ({0}) WHERE ""BlocksetID"" >= 0 ", scantableDefinition)))
while (rd.Read())
{
var id = rd.GetInt64(0);
var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc);
var filesize = rd.GetInt64(2);
var path = rd.GetString(3);
var metahash = rd.GetString(4);
var metasize = rd.GetInt64(5);
m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize));
}
}
if (m_pathLookup != null)
{
try
{
using (new Logging.Timer("Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = rd.GetInt64(1);
var metadataid = rd.GetInt64(2);
var filesetid = rd.GetInt64(3);
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate file entries detected, run repair to fix it", ex);
}
}
m_missingBlockHashes = cmd.ExecuteScalarInt64(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Block"".""VolumeID"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
if (tc > 0)
{
throw new InvalidDataException("Detected blocks that are not reachable in the block table");
}
}
}
public LocalRecreateDatabase(LocalDatabase parentdb, Options options)
: base(parentdb, options.Blocksize)
{
m_tempblocklist = "TempBlocklist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using (var cmd = m_connection.CreateCommand())
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist));
m_insertFileCommand = m_connection.CreateCommand();
m_insertFilesetEntryCommand = m_connection.CreateCommand();
m_insertMetadatasetCommand = m_connection.CreateCommand();
m_insertBlocksetCommand = m_connection.CreateCommand();
m_insertBlocklistHashCommand = m_connection.CreateCommand();
m_updateBlockVolumeCommand = m_connection.CreateCommand();
m_insertBlockset = m_connection.CreateCommand();
m_findBlocksetCommand = m_connection.CreateCommand();
m_findMetadatasetCommand = m_connection.CreateCommand();
m_findFilesetCommand = m_connection.CreateCommand();
m_findblocklisthashCommand = m_connection.CreateCommand();
m_findHashBlockCommand = m_connection.CreateCommand();
m_insertBlockCommand = m_connection.CreateCommand();
m_insertDuplicateBlockCommand = m_connection.CreateCommand();
m_insertFileCommand.CommandText = @"INSERT INTO ""File"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?); SELECT last_insert_rowid();";
m_insertFileCommand.AddParameters(3);
m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Scantime"") VALUES (?,?,?)";
m_insertFilesetEntryCommand.AddParameters(3);
m_insertMetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();";
m_insertMetadatasetCommand.AddParameters(1);
m_insertBlocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?,?); SELECT last_insert_rowid();";
m_insertBlocksetCommand.AddParameters(2);
m_insertBlocklistHashCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?,?,?)";
m_insertBlocklistHashCommand.AddParameters(3);
m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?";
m_updateBlockVolumeCommand.AddParameters(3);
m_insertBlockset.CommandText = string.Format(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist);
m_insertBlockset.AddParameters(3);
m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Size"" = ? AND ""FullHash"" = ? ";
m_findBlocksetCommand.AddParameters(2);
m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""BlocksetEntry"",""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? ";
m_findMetadatasetCommand.AddParameters(2);
m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? ";
m_findFilesetCommand.AddParameters(3);
m_findblocklisthashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist);
m_findblocklisthashCommand.AddParameters(1);
m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? ";
m_findHashBlockCommand.AddParameters(2);
m_insertBlockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (?,?,?)";
m_insertBlockCommand.AddParameters(3);
m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""Hash"", ""Size"", ""VolumeID"") VALUE (?,?,?)";
m_insertDuplicateBlockCommand.AddParameters(3);
if (options.BlockHashLookupMemory > 0)
{
m_blockHashLookup = new HashLookupHelper <long>((ulong)options.BlockHashLookupMemory / 2);
m_blockListHashLookup = new HashLookupHelper <bool>((ulong)options.BlockHashLookupMemory / 2);
}
if (options.FileHashLookupMemory > 0)
{
m_fileHashLookup = new HashLookupHelper <long>((ulong)options.FileHashLookupMemory);
}
if (options.MetadataHashMemory > 0)
{
m_metadataLookup = new HashLookupHelper <long>((ulong)options.MetadataHashMemory);
}
if (options.UseFilepathCache)
{
m_filesetLookup = new PathLookupHelper <PathEntryKeeper>();
}
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.BlockHashLookupMemory > 0)
{
m_blockHashLookup = new HashLookupHelper <KeyValuePair <long, long> >((ulong)options.BlockHashLookupMemory);
}
if (options.FileHashLookupMemory > 0)
{
m_fileHashLookup = new HashLookupHelper <long>((ulong)options.FileHashLookupMemory);
}
if (options.MetadataHashMemory > 0)
{
m_metadataLookup = new HashLookupHelper <long>((ulong)options.MetadataHashMemory);
}
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
if (m_blockHashLookup != null)
{
using (new Logging.Timer("Build blockhash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""ID"", ""Block"".""Size"" FROM ""Block"" "))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var id = Convert.ToInt64(rd.GetValue(1));
var size = Convert.ToInt64(rd.GetValue(2));
m_blockHashLookup.Add(str, size, new KeyValuePair <long, long>(id, size));
}
}
if (m_fileHashLookup != null)
{
using (new Logging.Timer("Build filehash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""Length"", ""ID"" FROM ""BlockSet"""))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var size = Convert.ToInt64(rd.GetValue(1));
var id = Convert.ToInt64(rd.GetValue(2));
m_fileHashLookup.Add(str, size, id);
}
}
if (m_metadataLookup != null)
{
using (new Logging.Timer("Build metahash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT ""Metadataset"".""ID"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""Metadataset"", ""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "))
while (rd.Read())
{
var metadataid = Convert.ToInt64(rd.GetValue(0));
var hash = rd.GetValue(1).ToString();
var size = Convert.ToInt64(rd.GetValue(2));
m_metadataLookup.Add(hash, size, metadataid);
}
}
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path scantime lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Scantime"", ""Path"", ""Scantime"" FROM ""{0}"" WHERE ""BlocksetID"" >= 0 ", m_scantimelookupTablename)))
while (rd.Read())
{
var id = Convert.ToInt64(rd.GetValue(0));
var scantime = ParseFromEpochSeconds(Convert.ToInt64(rd.GetValue(1)));
var path = rd.GetValue(2).ToString();
m_pathLookup.Insert(path, new PathEntryKeeper(id, scantime));
}
}
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = Convert.ToInt64(rd.GetValue(1));
var metadataid = Convert.ToInt64(rd.GetValue(2));
var filesetid = Convert.ToInt64(rd.GetValue(3));
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, DateTime.UtcNow);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
m_missingBlockHashes = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Block"".""VolumeID"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()));
}
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
//Need a temporary table with path/lastmodified lookups
var scantableDefinition =
@"SELECT ""A1"".""ID"" AS ""FileID"", ""A1"".""Lastmodified"" AS ""Lastmodified"", ""A1"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"", ""A1"".""BlocksetID"" " +
@" FROM (SELECT ""File"".""ID"", ""File"".""BlocksetID"", ""File"".""MetadataID"", ""FilesetEntry"".""Lastmodified"", ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A1"" LEFT JOIN " +
@" (SELECT ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A2"" ON ""A1"".""Path"" = ""A2"".""Path"" AND ""A1"".""Timestamp"" < ""A2"".""Timestamp"" " +
@" , ""Blockset"" ""C"", ""Metadataset"" ""E"", ""Blockset"" ""F"" " +
@" WHERE ""A2"".""Path"" IS NULL " +
@" AND ""C"".""ID"" = ""A1"".""BlocksetID"" " +
@" AND ""A1"".""MetadataID"" = ""E"".""ID"" " +
@" AND ""F"".""ID"" = ""E"".""BlocksetID"" ";
if (m_pathLookup != null)
{
using (new Logging.Timer(LOGTAG, "BuildLastModified", "Build path lastmodified lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ({0}) WHERE ""BlocksetID"" >= 0 ", scantableDefinition)))
while (rd.Read())
{
var id = rd.GetInt64(0);
var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc);
var filesize = rd.GetInt64(2);
var path = rd.GetString(3);
var metahash = rd.GetString(4);
var metasize = rd.GetInt64(5);
m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize));
}
}
if (m_pathLookup != null)
{
try
{
using (new Logging.Timer(LOGTAG, "BuildPathTable", "Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = rd.GetInt64(1);
var metadataid = rd.GetInt64(2);
var filesetid = rd.GetInt64(3);
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate file entries detected, run repair to fix it", ex);
}
}
var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
if (tc > 0)
{
throw new InvalidDataException("Detected blocks that are not reachable in the block table");
}
}
if (options.UseBlockCache)
{
string failedhash = null;
try
{
var cache = new Dictionary <string, long>();
using (var cmd = m_connection.CreateCommand())
{
cmd.CommandText = @"SELECT ""Hash"", ""Size"" From ""Block""";
using (var rd = cmd.ExecuteReader())
while (rd.Read())
{
cache.Add(failedhash = rd.ConvertValueToString(0), rd.ConvertValueToInt64(1));
}
}
m_blockCache = cache;
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheFailure", ex, "Failed to create block cache, this could mean you have hash collisions in your table, the hash that failed is {0}. Error message: {1}.", failedhash, ex.Message);
Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheFailure", null, "Disabling block cache due to error");
}
}
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.BlockHashLookupMemory > 0)
{
m_blockHashLookup = new HashLookupHelper <KeyValuePair <long, long> >((ulong)options.BlockHashLookupMemory);
}
if (options.FileHashLookupMemory > 0)
{
m_fileHashLookup = new HashLookupHelper <long>((ulong)options.FileHashLookupMemory);
}
if (options.MetadataHashMemory > 0)
{
m_metadataLookup = new HashLookupHelper <long>((ulong)options.MetadataHashMemory);
}
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
//Need a temporary table with path/lastmodified lookups
var scantableDefinition = @"SELECT ""B"".""ID"" AS ""FileID"", ""D"".""Lastmodified"" AS ""Lastmodified"", ""B"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"" FROM (SELECT ""FilesetEntry"".""FileID"" AS ""FileID"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"", MAX(""Fileset"".""Timestamp"") AS ""MostRecent"" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" GROUP BY ""File"".""Path"") A, ""File"" B, ""Blockset"" C, ""FilesetEntry"" D, ""Metadataset"" E, ""Blockset"" F WHERE ""B"".""ID"" = ""A"".""FileID"" AND ""C"".""ID"" = ""B"".""BlocksetID"" AND ""D"".""FileID"" = ""B"".""ID"" AND ""D"".""FilesetID"" = ""A"".""FilesetID"" AND ""B"".""MetadataID"" = ""E"".""ID"" AND ""F"".""ID"" = ""E"".""BlocksetID""";
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + scantableDefinition, m_lastmodifiedLookupTablename));
cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}Index"" ON ""{0}"" (""Path"", ""Lastmodified"", ""Length"", ""Metahash"", ""Metasize"", ""FileID"") ", m_lastmodifiedLookupTablename));
if (m_blockHashLookup != null)
{
try
{
using (new Logging.Timer("Build blockhash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""ID"", ""Block"".""Size"" FROM ""Block"" "))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var id = rd.GetInt64(1);
var size = rd.GetInt64(2);
m_blockHashLookup.Add(str, size, new KeyValuePair <long, long>(id, size));
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate blockhashes detected, either repair the database or rebuild it", ex);
}
}
if (m_fileHashLookup != null)
{
try
{
using (new Logging.Timer("Build filehash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""Length"", ""ID"" FROM ""BlockSet"""))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var size = rd.GetInt64(1);
var id = rd.GetInt64(2);
m_fileHashLookup.Add(str, size, id);
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate filehashes detected, either repair the database or rebuild it", ex);
}
}
if (m_metadataLookup != null)
{
try
{
using (new Logging.Timer("Build metahash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT ""Metadataset"".""ID"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""Metadataset"", ""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "))
while (rd.Read())
{
var metadataid = rd.GetInt64(0);
var hash = rd.GetValue(1).ToString();
var size = rd.GetInt64(2);;
m_metadataLookup.Add(hash, size, metadataid);
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate metadatahash detected, run repair to fix it", ex);
}
}
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path lastmodified lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ""{0}"" WHERE ""BlocksetID"" >= 0 ", m_lastmodifiedLookupTablename)))
while (rd.Read())
{
var id = rd.GetInt64(0);
var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc);
var filesize = rd.GetInt64(2);
var path = rd.GetString(3);
var metahash = rd.GetString(4);
var metasize = rd.GetInt64(5);
m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize));
}
}
if (m_pathLookup != null)
{
try
{
using (new Logging.Timer("Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = rd.GetInt64(1);
var metadataid = rd.GetInt64(2);
var filesetid = rd.GetInt64(3);
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate file entries detected, run repair to fix it", ex);
}
}
m_missingBlockHashes = cmd.ExecuteScalarInt64(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Block"".""VolumeID"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
if (tc > 0)
{
throw new InvalidDataException("Detected blocks that are not reachable in the block table");
}
}
}
public LocalRecreateDatabase(LocalDatabase parentdb, Options options)
: base(parentdb)
{
m_tempblocklist = "TempBlocklist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
m_tempsmalllist = "TempSmalllist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using (var cmd = m_connection.CreateCommand())
{
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist));
cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""Index_{0}"" ON ""{0}"" (""BlockListHash"");", m_tempblocklist));
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)", m_tempsmalllist));
cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_File_{0}"" ON ""{0}"" (""FileHash"", ""BlockSize"");", m_tempsmalllist));
cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_Block_{0}"" ON ""{0}"" (""BlockHash"", ""BlockSize"");", m_tempsmalllist));
}
m_insertFileCommand = m_connection.CreateCommand();
m_insertFilesetEntryCommand = m_connection.CreateCommand();
m_insertMetadatasetCommand = m_connection.CreateCommand();
m_insertBlocksetCommand = m_connection.CreateCommand();
m_insertBlocklistHashCommand = m_connection.CreateCommand();
m_updateBlockVolumeCommand = m_connection.CreateCommand();
m_insertBlockset = m_connection.CreateCommand();
m_insertSmallBlockset = m_connection.CreateCommand();
m_findBlocksetCommand = m_connection.CreateCommand();
m_findMetadatasetCommand = m_connection.CreateCommand();
m_findFilesetCommand = m_connection.CreateCommand();
m_findblocklisthashCommand = m_connection.CreateCommand();
m_findHashBlockCommand = m_connection.CreateCommand();
m_insertBlockCommand = m_connection.CreateCommand();
m_insertDuplicateBlockCommand = m_connection.CreateCommand();
m_insertFileCommand.CommandText = @"INSERT INTO ""File"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?); SELECT last_insert_rowid();";
m_insertFileCommand.AddParameters(3);
m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?,?,?)";
m_insertFilesetEntryCommand.AddParameters(3);
m_insertMetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();";
m_insertMetadatasetCommand.AddParameters(1);
m_insertBlocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?,?); SELECT last_insert_rowid();";
m_insertBlocksetCommand.AddParameters(2);
m_insertBlocklistHashCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?,?,?)";
m_insertBlocklistHashCommand.AddParameters(3);
m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?";
m_updateBlockVolumeCommand.AddParameters(3);
m_insertBlockset.CommandText = string.Format(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist);
m_insertBlockset.AddParameters(3);
m_insertSmallBlockset.CommandText = string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) ", m_tempsmalllist);
m_insertSmallBlockset.AddParameters(3);
m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? ";
m_findBlocksetCommand.AddParameters(2);
m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = ? AND ""Blockset"".""Length"" = ? ";
m_findMetadatasetCommand.AddParameters(2);
m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? ";
m_findFilesetCommand.AddParameters(3);
m_findblocklisthashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist);
m_findblocklisthashCommand.AddParameters(1);
m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? ";
m_findHashBlockCommand.AddParameters(2);
m_insertBlockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (?,?,?)";
m_insertBlockCommand.AddParameters(3);
m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?), ?)";
m_insertDuplicateBlockCommand.AddParameters(3);
if (options.UseFilepathCache)
{
m_filesetLookup = new PathLookupHelper <PathEntryKeeper>();
}
}