public override void Dispose()
{
m_fileHashLookup = null;
m_metadataLookup = null;
m_blockHashLookup = null;
m_pathLookup = null;
base.Dispose();
}
public void Dispose()
{
m_lookup = null;
if (m_command != null)
{
try { m_command.Dispose(); }
finally { m_command = null; }
}
}
public BlockQuery(System.Data.IDbConnection con, Options options, System.Data.IDbTransaction transaction)
{
m_command = con.CreateCommand();
m_command.Transaction = transaction;
if (options.BlockHashLookupMemory > 0)
{
m_lookup = new HashLookupHelper <long>((ulong)options.BlockHashLookupMemory);
using (var reader = m_command.ExecuteReader(@"SELECT ""Hash"", ""Size"" FROM ""Block"" "))
while (reader.Read())
{
var hash = reader.GetString(0);
var size = reader.GetInt64(1);
m_lookup.Add(hash, size, size);
}
}
m_command.Parameters.Clear();
m_command.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? ";
m_command.AddParameters(2);
}
public override void Dispose()
{
if (m_scantimelookupTablename != null)
{
try
{
using (var cmd = m_connection.CreateCommand())
cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_scantimelookupTablename));
}
catch { }
finally
{
m_scantimelookupTablename = null;
}
}
m_fileHashLookup = null;
m_metadataLookup = null;
m_blockHashLookup = null;
m_pathLookup = null;
base.Dispose();
}
public void Dispose()
{
m_lookup = null;
if (m_command != null)
try { m_command.Dispose(); }
finally { m_command = null; }
}
public BlockQuery(System.Data.IDbConnection con, Options options, System.Data.IDbTransaction transaction)
{
m_command = con.CreateCommand();
m_command.Transaction = transaction;
if (options.BlockHashLookupMemory > 0)
{
m_lookup = new HashLookupHelper<long>((ulong)options.BlockHashLookupMemory);
using(var reader = m_command.ExecuteReader(@"SELECT ""Hash"", ""Size"" FROM ""Block"" "))
while (reader.Read())
{
var hash = reader.GetValue(0).ToString();
var size = Convert.ToInt64(reader.GetValue(1));
m_lookup.Add(hash, size, size);
}
}
m_command.Parameters.Clear();
m_command.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? ";
m_command.AddParameters(2);
}
public static int Run(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count != 2 && args.Count != 3)
{
Console.WriteLine("Invalid argument count ({0} expected 2 or 3): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args));
return(100);
}
var folder = Path.GetFullPath(args[1]);
if (!Directory.Exists(folder))
{
Console.WriteLine("Folder not found: {0}", folder);
return(100);
}
Directory.SetCurrentDirectory(folder);
string targetpath;
options.TryGetValue("targetpath", out targetpath);
string ixfile;
options.TryGetValue("indexfile", out ixfile);
if (string.IsNullOrWhiteSpace(ixfile))
{
ixfile = "index.txt";
}
ixfile = Path.GetFullPath(ixfile);
if (!File.Exists(ixfile))
{
Console.WriteLine("Index file not found, perhaps you need to run the index command?");
return(100);
}
Console.Write("Sorting index file ...");
Index.SortFile(ixfile, ixfile);
Console.WriteLine(" done!");
string filelist;
if (args.Count == 2)
{
var time = List.ParseListFiles(folder).First();
filelist = time.Value;
Console.WriteLine("Using set 0 with timestamp {0}", time.Key.ToLocalTime());
}
else
{
filelist = List.SelectListFile(args[2], folder);
}
Library.Main.Volumes.VolumeReaderBase.UpdateOptionsFromManifest(Path.GetExtension(filelist).Trim('.'), filelist, new Duplicati.Library.Main.Options(options));
string blocksize_str;
options.TryGetValue("blocksize", out blocksize_str);
string blockhash_str;
options.TryGetValue("block-hash-algorithm", out blockhash_str);
string filehash_str;
options.TryGetValue("block-hash-algorithm", out filehash_str);
long blocksize = string.IsNullOrWhiteSpace(blocksize_str) ? 0 : Library.Utility.Sizeparser.ParseSize(blocksize_str);
if (blocksize <= 0)
{
Console.WriteLine("Invalid blocksize: {0}, try setting --blocksize manually");
return(100);
}
var blockhasher = string.IsNullOrWhiteSpace(blockhash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(blockhash_str);
var filehasher = string.IsNullOrWhiteSpace(filehash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(filehash_str);
if (blockhasher == null)
{
throw new Duplicati.Library.Interface.UserInformationException(string.Format("Block hash algorithm not valid: {0}", blockhash_str));
}
if (filehasher == null)
{
throw new Duplicati.Library.Interface.UserInformationException(string.Format("File hash algorithm not valid: {0}", filehash_str));
}
var hashesprblock = blocksize / (blockhasher.HashSize / 8);
using (var mru = new CompressedFileMRUCache(options))
{
Console.WriteLine("Building lookup table for file hashes");
var lookup = new HashLookupHelper(ixfile, mru, (int)blocksize, blockhasher.HashSize / 8);
var filecount = 0L;
string largestprefix = null;
string[] largestprefixparts = null;
if (!string.IsNullOrWhiteSpace(targetpath))
{
Console.WriteLine("Computing restore path");
}
foreach (var f in List.EnumerateFilesInDList(filelist, filter, options))
{
if (largestprefix == null)
{
largestprefix = f.Path;
largestprefixparts = largestprefix.Split(new char[] { Path.DirectorySeparatorChar });
}
else if (largestprefix.Length > 1)
{
var parts = f.Path.Split(new char[] { Path.DirectorySeparatorChar });
var ni = 0;
for (; ni < Math.Min(parts.Length, largestprefixparts.Length); ni++)
{
if (!Library.Utility.Utility.ClientFilenameStringComparer.Equals(parts[ni], largestprefixparts[ni]))
{
break;
}
}
if (ni != largestprefixparts.Length)
{
if (ni == 0)
{
largestprefixparts = new string[0];
largestprefix = string.Empty;
}
else
{
Array.Resize(ref largestprefixparts, ni - 1);
largestprefix = string.Join(Path.DirectorySeparatorChar.ToString(), largestprefixparts);
}
}
}
filecount++;
}
Console.WriteLine("Restoring {0} files to {1}", filecount, string.IsNullOrWhiteSpace(targetpath) ? "original position" : targetpath);
if (Library.Utility.Utility.IsClientLinux || largestprefix.Length > 0)
{
largestprefix = Library.Utility.Utility.AppendDirSeparator(largestprefix);
}
if (!string.IsNullOrEmpty(largestprefix))
{
Console.WriteLine("Removing common prefix {0} from files", largestprefix);
}
var i = 0L;
var errors = 0L;
foreach (var f in List.EnumerateFilesInDList(filelist, filter, options))
{
try
{
var targetfile = MapToRestorePath(f.Path, largestprefix, targetpath);
if (!Directory.Exists(Path.GetDirectoryName(targetfile)))
{
Directory.CreateDirectory(Path.GetDirectoryName(targetfile));
}
Console.Write("{0}: {1} ({2})", i, targetfile, Library.Utility.Utility.FormatSizeString(f.Size));
using (var tf = new Library.Utility.TempFile())
{
using (var sw = File.OpenWrite(tf))
{
if (f.BlocklistHashes == null)
{
lookup.WriteHash(sw, f.Hash);
}
else
{
var blhi = 0L;
foreach (var blh in f.BlocklistHashes)
{
Console.Write(" {0}", blhi);
var blockhashoffset = blhi * hashesprblock * blocksize;
try
{
var bi = 0;
foreach (var h in lookup.ReadBlocklistHashes(blh))
{
try
{
sw.Position = blockhashoffset + (bi * blocksize);
lookup.WriteHash(sw, h);
}
catch (Exception ex)
{
Console.WriteLine("Failed to read hash: {0}{1}{2}", h, Environment.NewLine, ex.ToString());
}
bi++;
}
}
catch (Exception ex)
{
Console.WriteLine("Failed to read Blocklist hash: {0}{1}{2}", blh, Environment.NewLine, ex.ToString());
}
blhi++;
}
}
}
string fh;
using (var fs = File.OpenRead(tf))
fh = Convert.ToBase64String(filehasher.ComputeHash(fs));
if (fh == f.Hash)
{
Console.WriteLine(" done!");
File.Copy(tf, targetfile, true);
}
else
{
Console.Write(" - Restored file hash mismatch");
if (File.Exists(targetfile))
{
Console.WriteLine(" - not overwriting existing file: {0}", targetfile);
}
else
{
Console.WriteLine(" - restoring file in damaged condition");
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(" error: {0}", ex.ToString());
errors++;
}
i++;
}
}
return(0);
}
public LocalRecreateDatabase(LocalDatabase parentdb, Options options)
: base(parentdb, options.Blocksize)
{
m_tempblocklist = "TempBlocklist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using (var cmd = m_connection.CreateCommand())
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist));
m_insertFileCommand = m_connection.CreateCommand();
m_insertFilesetEntryCommand = m_connection.CreateCommand();
m_insertMetadatasetCommand = m_connection.CreateCommand();
m_insertBlocksetCommand = m_connection.CreateCommand();
m_insertBlocklistHashCommand = m_connection.CreateCommand();
m_updateBlockVolumeCommand = m_connection.CreateCommand();
m_insertBlockset = m_connection.CreateCommand();
m_findBlocksetCommand = m_connection.CreateCommand();
m_findMetadatasetCommand = m_connection.CreateCommand();
m_findFilesetCommand = m_connection.CreateCommand();
m_findblocklisthashCommand = m_connection.CreateCommand();
m_findHashBlockCommand = m_connection.CreateCommand();
m_insertBlockCommand = m_connection.CreateCommand();
m_insertDuplicateBlockCommand = m_connection.CreateCommand();
m_insertFileCommand.CommandText = @"INSERT INTO ""File"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?); SELECT last_insert_rowid();";
m_insertFileCommand.AddParameters(3);
m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Scantime"") VALUES (?,?,?)";
m_insertFilesetEntryCommand.AddParameters(3);
m_insertMetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();";
m_insertMetadatasetCommand.AddParameters(1);
m_insertBlocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?,?); SELECT last_insert_rowid();";
m_insertBlocksetCommand.AddParameters(2);
m_insertBlocklistHashCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?,?,?)";
m_insertBlocklistHashCommand.AddParameters(3);
m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?";
m_updateBlockVolumeCommand.AddParameters(3);
m_insertBlockset.CommandText = string.Format(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist);
m_insertBlockset.AddParameters(3);
m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Size"" = ? AND ""FullHash"" = ? ";
m_findBlocksetCommand.AddParameters(2);
m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""BlocksetEntry"",""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? ";
m_findMetadatasetCommand.AddParameters(2);
m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? ";
m_findFilesetCommand.AddParameters(3);
m_findblocklisthashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist);
m_findblocklisthashCommand.AddParameters(1);
m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? ";
m_findHashBlockCommand.AddParameters(2);
m_insertBlockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (?,?,?)";
m_insertBlockCommand.AddParameters(3);
m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""Hash"", ""Size"", ""VolumeID"") VALUE (?,?,?)";
m_insertDuplicateBlockCommand.AddParameters(3);
if (options.BlockHashLookupMemory > 0)
{
m_blockHashLookup = new HashLookupHelper <long>((ulong)options.BlockHashLookupMemory / 2);
m_blockListHashLookup = new HashLookupHelper <bool>((ulong)options.BlockHashLookupMemory / 2);
}
if (options.FileHashLookupMemory > 0)
{
m_fileHashLookup = new HashLookupHelper <long>((ulong)options.FileHashLookupMemory);
}
if (options.MetadataHashMemory > 0)
{
m_metadataLookup = new HashLookupHelper <long>((ulong)options.MetadataHashMemory);
}
if (options.UseFilepathCache)
{
m_filesetLookup = new PathLookupHelper <PathEntryKeeper>();
}
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.BlockHashLookupMemory > 0)
{
m_blockHashLookup = new HashLookupHelper <KeyValuePair <long, long> >((ulong)options.BlockHashLookupMemory);
}
if (options.FileHashLookupMemory > 0)
{
m_fileHashLookup = new HashLookupHelper <long>((ulong)options.FileHashLookupMemory);
}
if (options.MetadataHashMemory > 0)
{
m_metadataLookup = new HashLookupHelper <long>((ulong)options.MetadataHashMemory);
}
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
if (m_blockHashLookup != null)
{
using (new Logging.Timer("Build blockhash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""ID"", ""Block"".""Size"" FROM ""Block"" "))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var id = Convert.ToInt64(rd.GetValue(1));
var size = Convert.ToInt64(rd.GetValue(2));
m_blockHashLookup.Add(str, size, new KeyValuePair <long, long>(id, size));
}
}
if (m_fileHashLookup != null)
{
using (new Logging.Timer("Build filehash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""Length"", ""ID"" FROM ""BlockSet"""))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var size = Convert.ToInt64(rd.GetValue(1));
var id = Convert.ToInt64(rd.GetValue(2));
m_fileHashLookup.Add(str, size, id);
}
}
if (m_metadataLookup != null)
{
using (new Logging.Timer("Build metahash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT ""Metadataset"".""ID"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""Metadataset"", ""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "))
while (rd.Read())
{
var metadataid = Convert.ToInt64(rd.GetValue(0));
var hash = rd.GetValue(1).ToString();
var size = Convert.ToInt64(rd.GetValue(2));
m_metadataLookup.Add(hash, size, metadataid);
}
}
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path scantime lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Scantime"", ""Path"", ""Scantime"" FROM ""{0}"" WHERE ""BlocksetID"" >= 0 ", m_scantimelookupTablename)))
while (rd.Read())
{
var id = Convert.ToInt64(rd.GetValue(0));
var scantime = ParseFromEpochSeconds(Convert.ToInt64(rd.GetValue(1)));
var path = rd.GetValue(2).ToString();
m_pathLookup.Insert(path, new PathEntryKeeper(id, scantime));
}
}
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = Convert.ToInt64(rd.GetValue(1));
var metadataid = Convert.ToInt64(rd.GetValue(2));
var filesetid = Convert.ToInt64(rd.GetValue(3));
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, DateTime.UtcNow);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
m_missingBlockHashes = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Block"".""VolumeID"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()));
}
}
public static int Run(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter)
{
if (args.Count != 2 && args.Count != 3)
{
Console.WriteLine("Invalid argument count ({0} expected 2 or 3): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args));
return 100;
}
var folder = Path.GetFullPath(args[1]);
if (!Directory.Exists(folder))
{
Console.WriteLine("Folder not found: {0}", folder);
return 100;
}
Directory.SetCurrentDirectory(folder);
string targetpath;
options.TryGetValue("targetpath", out targetpath);
string ixfile;
options.TryGetValue("indexfile", out ixfile);
if (string.IsNullOrWhiteSpace(ixfile))
ixfile = "index.txt";
ixfile = Path.GetFullPath(ixfile);
if (!File.Exists(ixfile))
{
Console.WriteLine("Index file not found, perhaps you need to run the index command?");
return 100;
}
Console.Write("Sorting index file ...");
Index.SortFile(ixfile, ixfile);
Console.WriteLine(" done!");
string filelist;
if (args.Count == 2)
{
var time = List.ParseListFiles(folder).First();
filelist = time.Value;
Console.WriteLine("Using set 0 with timestamp {0}", time.Key.ToLocalTime());
}
else
{
filelist = List.SelectListFile(args[2], folder);
}
Library.Main.Volumes.VolumeReaderBase.UpdateOptionsFromManifest(Path.GetExtension(filelist).Trim('.'), filelist, new Duplicati.Library.Main.Options(options));
string blocksize_str;
options.TryGetValue("blocksize", out blocksize_str);
string blockhash_str;
options.TryGetValue("block-hash-algorithm", out blockhash_str);
string filehash_str;
options.TryGetValue("block-hash-algorithm", out filehash_str);
long blocksize = string.IsNullOrWhiteSpace(blocksize_str) ? 0 : Library.Utility.Sizeparser.ParseSize(blocksize_str);
if (blocksize <= 0)
{
Console.WriteLine("Invalid blocksize: {0}, try setting --blocksize manually");
return 100;
}
var blockhasher = string.IsNullOrWhiteSpace(blockhash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(blockhash_str);
var filehasher = string.IsNullOrWhiteSpace(filehash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(filehash_str);
if (blockhasher == null)
throw new Exception(string.Format("Block hash algorithm not valid: {0}", blockhash_str));
if (filehasher == null)
throw new Exception(string.Format("File hash algorithm not valid: {0}", filehash_str));
var hashesprblock = blocksize / (blockhasher.HashSize / 8);
using(var mru = new CompressedFileMRUCache(options))
{
Console.WriteLine("Building lookup table for file hashes");
var lookup = new HashLookupHelper(ixfile, mru, (int)blocksize, blockhasher.HashSize / 8);
var filecount = 0L;
string largestprefix = null;
string[] largestprefixparts = null;
if (!string.IsNullOrWhiteSpace(targetpath))
Console.WriteLine("Computing restore path");
foreach(var f in List.EnumerateFilesInDList(filelist, filter, options))
{
if (largestprefix == null)
{
largestprefix = f.Path;
largestprefixparts = largestprefix.Split(new char[] { Path.DirectorySeparatorChar });
}
else if (largestprefix.Length > 1)
{
var parts = f.Path.Split(new char[] { Path.DirectorySeparatorChar });
var ni = 0;
for(; ni < Math.Min(parts.Length, largestprefixparts.Length); ni++)
if (!Library.Utility.Utility.ClientFilenameStringComparer.Equals(parts[ni], largestprefixparts[ni]))
break;
if (ni != largestprefixparts.Length)
{
if (ni == 0)
{
largestprefixparts = new string[0];
largestprefix = string.Empty;
}
else
{
Array.Resize(ref largestprefixparts, ni - 1);
largestprefix = string.Join(Path.DirectorySeparatorChar.ToString(), largestprefixparts);
}
}
}
filecount++;
}
Console.WriteLine("Restoring {0} files to {1}", filecount, string.IsNullOrWhiteSpace(targetpath) ? "original position" : targetpath);
if (Library.Utility.Utility.IsClientLinux || largestprefix.Length > 0)
largestprefix = Library.Utility.Utility.AppendDirSeparator(largestprefix);
if (!string.IsNullOrEmpty(largestprefix))
Console.WriteLine("Removing common prefix {0} from files", largestprefix);
var i = 0L;
var errors = 0L;
foreach(var f in List.EnumerateFilesInDList(filelist, filter, options))
{
try
{
var targetfile = MapToRestorePath(f.Path, largestprefix, targetpath);
if (!Directory.Exists(Path.GetDirectoryName(targetfile)))
Directory.CreateDirectory(Path.GetDirectoryName(targetfile));
Console.Write("{0}: {1} ({2})", i, targetfile, Library.Utility.Utility.FormatSizeString(f.Size));
using(var tf = new Library.Utility.TempFile())
{
using(var sw = File.OpenWrite(tf))
{
if (f.BlocklistHashes == null)
{
lookup.WriteHash(sw, f.Hash);
}
else
{
var blhi = 0L;
foreach(var blh in f.BlocklistHashes)
{
Console.Write(" {0}", blhi);
var blockhashoffset = blhi * hashesprblock * blocksize;
try
{
var bi = 0;
foreach(var h in lookup.ReadBlocklistHashes(blh))
{
try
{
sw.Position = blockhashoffset + (bi * blocksize);
lookup.WriteHash(sw, h);
}
catch(Exception ex)
{
Console.WriteLine("Failed to read hash: {0}{1}{2}", h, Environment.NewLine, ex.ToString());
}
bi++;
}
}
catch (Exception ex)
{
Console.WriteLine("Failed to read Blocklist hash: {0}{1}{2}", blh, Environment.NewLine, ex.ToString());
}
blhi++;
}
}
}
string fh;
using(var fs = File.OpenRead(tf))
fh = Convert.ToBase64String(filehasher.ComputeHash(fs));
if (fh == f.Hash)
{
Console.WriteLine(" done!");
File.Copy(tf, targetfile, true);
}
else
{
Console.Write(" - Restored file hash mismatch");
if (File.Exists(targetfile))
Console.WriteLine(" - not overwriting existing file: {0}", targetfile);
else
Console.WriteLine(" - restoring file in damaged condition");
}
}
}
catch (Exception ex)
{
Console.WriteLine(" error: {0}", ex.ToString());
errors++;
}
i++;
}
}
return 0;
}
/// <summary>
/// Builds the lookup tables. Call this method after deleting items, and before processing items
/// </summary>
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.BlockHashLookupMemory > 0)
{
m_blockHashLookup = new HashLookupHelper <KeyValuePair <long, long> >((ulong)options.BlockHashLookupMemory);
}
if (options.FileHashLookupMemory > 0)
{
m_fileHashLookup = new HashLookupHelper <long>((ulong)options.FileHashLookupMemory);
}
if (options.MetadataHashMemory > 0)
{
m_metadataLookup = new HashLookupHelper <long>((ulong)options.MetadataHashMemory);
}
if (options.UseFilepathCache)
{
m_pathLookup = new PathLookupHelper <PathEntryKeeper>(true);
}
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
//Need a temporary table with path/lastmodified lookups
var scantableDefinition =
@"SELECT ""A1"".""ID"" AS ""FileID"", ""A1"".""Lastmodified"" AS ""Lastmodified"", ""A1"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"", ""A1"".""BlocksetID"" " +
@" FROM (SELECT ""File"".""ID"", ""File"".""BlocksetID"", ""File"".""MetadataID"", ""FilesetEntry"".""Lastmodified"", ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A1"" LEFT JOIN " +
@" (SELECT ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A2"" ON ""A1"".""Path"" = ""A2"".""Path"" AND ""A1"".""Timestamp"" < ""A2"".""Timestamp"" " +
@" , ""Blockset"" ""C"", ""Metadataset"" ""E"", ""Blockset"" ""F"" " +
@" WHERE ""A2"".""Path"" IS NULL " +
@" AND ""C"".""ID"" = ""A1"".""BlocksetID"" " +
@" AND ""A1"".""MetadataID"" = ""E"".""ID"" " +
@" AND ""F"".""ID"" = ""E"".""BlocksetID"" ";
if (m_blockHashLookup != null)
{
try
{
using (new Logging.Timer("Build blockhash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""ID"", ""Block"".""Size"" FROM ""Block"" "))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var id = rd.GetInt64(1);
var size = rd.GetInt64(2);
m_blockHashLookup.Add(str, size, new KeyValuePair <long, long>(id, size));
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate blockhashes detected, either repair the database or rebuild it", ex);
}
}
if (m_fileHashLookup != null)
{
try
{
using (new Logging.Timer("Build filehash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""Length"", ""ID"" FROM ""BlockSet"""))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var size = rd.GetInt64(1);
var id = rd.GetInt64(2);
m_fileHashLookup.Add(str, size, id);
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate filehashes detected, either repair the database or rebuild it", ex);
}
}
if (m_metadataLookup != null)
{
try
{
using (new Logging.Timer("Build metahash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT ""Metadataset"".""ID"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""Metadataset"", ""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "))
while (rd.Read())
{
var metadataid = rd.GetInt64(0);
var hash = rd.GetValue(1).ToString();
var size = rd.GetInt64(2);;
m_metadataLookup.Add(hash, size, metadataid);
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate metadatahash detected, run repair to fix it", ex);
}
}
if (m_pathLookup != null)
{
using (new Logging.Timer("Build path lastmodified lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ({0}) WHERE ""BlocksetID"" >= 0 ", scantableDefinition)))
while (rd.Read())
{
var id = rd.GetInt64(0);
var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc);
var filesize = rd.GetInt64(2);
var path = rd.GetString(3);
var metahash = rd.GetString(4);
var metasize = rd.GetInt64(5);
m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize));
}
}
if (m_pathLookup != null)
{
try
{
using (new Logging.Timer("Build path lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = rd.GetInt64(1);
var metadataid = rd.GetInt64(2);
var filesetid = rd.GetInt64(3);
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
{
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate file entries detected, run repair to fix it", ex);
}
}
m_missingBlockHashes = cmd.ExecuteScalarInt64(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Block"".""VolumeID"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
if (tc > 0)
{
throw new InvalidDataException("Detected blocks that are not reachable in the block table");
}
}
}