/// <summary>
/// 开始事务 递数据库连接生成数据库事务
/// </summary>
public static void BeginTransaction()
{
try
{
Trans = Mder.FC.DB.Conn.Instance.BeginTransaction(System .Data .IsolationLevel .ReadCommitted );
}
catch (Exception ex)
{
Err = ex.Message;
RollBack();
//如果再出错，说明代码写的有问题，将把错误抛给前端
Trans = Mder.FC.DB.Conn.Instance.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
}
}
public long AddBlockset(string fullhash, long size, IEnumerable <string> blocklisthashes, long expectedblocklisthashes, System.Data.IDbTransaction transaction)
{
m_findBlocksetCommand.Transaction = transaction;
m_findBlocksetCommand.SetParameterValue(0, size);
m_findBlocksetCommand.SetParameterValue(1, fullhash);
var blocksetid = m_findBlocksetCommand.ExecuteScalarInt64(-1);
if (blocksetid != -1)
{
return(blocksetid);
}
m_insertBlocksetCommand.Transaction = transaction;
m_insertBlocksetCommand.SetParameterValue(0, size);
m_insertBlocksetCommand.SetParameterValue(1, fullhash);
blocksetid = m_insertBlocksetCommand.ExecuteScalarInt64(-1);
long c = 0;
if (blocklisthashes != null)
{
var index = 0L;
m_insertBlocklistHashCommand.Transaction = transaction;
m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid);
foreach (var hash in blocklisthashes)
{
if (!string.IsNullOrEmpty(hash))
{
c++;
if (c <= expectedblocklisthashes)
{
m_insertBlocklistHashCommand.SetParameterValue(1, index++);
m_insertBlocklistHashCommand.SetParameterValue(2, hash);
m_insertBlocklistHashCommand.ExecuteNonQuery();
}
}
}
}
if (c != expectedblocklisthashes)
{
Logging.Log.WriteWarningMessage(LOGTAG, "MismatchInBlocklistHashCount", null, "Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid);
}
return(blocksetid);
}
private void AddEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
{
var fileid = -1L;
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
{
fileid = e.GetFilesetID(blocksetid, metadataid);
}
}
else
{
m_findFilesetCommand.Transaction = transaction;
m_findFilesetCommand.SetParameterValue(0, path);
m_findFilesetCommand.SetParameterValue(1, blocksetid);
m_findFilesetCommand.SetParameterValue(2, metadataid);
fileid = m_findFilesetCommand.ExecuteScalarInt64(-1);
}
if (fileid < 0)
{
m_insertFileCommand.Transaction = transaction;
m_insertFileCommand.SetParameterValue(0, path);
m_insertFileCommand.SetParameterValue(1, blocksetid);
m_insertFileCommand.SetParameterValue(2, metadataid);
fileid = m_insertFileCommand.ExecuteScalarInt64(-1);
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
{
e.AddFilesetID(blocksetid, metadataid, fileid);
}
else
{
e = new PathEntryKeeper();
e.AddFilesetID(blocksetid, metadataid, fileid);
m_filesetLookup.Insert(path, e);
}
}
}
m_insertFilesetEntryCommand.Transaction = transaction;
m_insertFilesetEntryCommand.SetParameterValue(0, filesetid);
m_insertFilesetEntryCommand.SetParameterValue(1, fileid);
m_insertFilesetEntryCommand.SetParameterValue(2, time.ToUniversalTime().Ticks);
m_insertFilesetEntryCommand.ExecuteNonQuery();
}
private bool Deleteitem(iSqlTransaction trans, string id)
{
try
{
if (trans == null || CFunctions.IsNullOrEmpty(id)) return false;
string SQL = "DELETE FROM " + TABLENAME + " WHERE id IN (" + id + ")";
HELPER.executeNonQuery(trans, SQL);
return true;
}
catch (Exception ex)
{
throw ex;
}
}
private bool Deleteitem(iSqlTransaction trans, string iid)
{
try
{
if (trans == null || string.IsNullOrEmpty(iid)) return false;
string SQL = "UPDATE " + TABLENAME + " SET status=" + (int)CConstants.State.Status.Deleted + " WHERE id IN (" + iid + ")";
HELPER.executeNonQuery(trans, SQL);
return true;
}
catch (Exception ex)
{
throw ex;
}
}
public StorageHelper(System.Data.IDbConnection con)
{
m_connection = con;
m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
m_transaction = m_connection.BeginTransaction();
using(var cmd = m_connection.CreateCommand())
{
cmd.Transaction = m_transaction;
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_previousTable));
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_currentTable));
}
m_insertPreviousElementCommand = m_connection.CreateCommand();
m_insertPreviousElementCommand.Transaction = m_transaction;
m_insertPreviousElementCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_previousTable);
m_insertPreviousElementCommand.AddParameters(5);
m_insertCurrentElementCommand = m_connection.CreateCommand();
m_insertCurrentElementCommand.Transaction = m_transaction;
m_insertCurrentElementCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_currentTable);
m_insertCurrentElementCommand.AddParameters(5);
}
public void UpdateBlockset(string hash, IEnumerable <string> blocklisthashes, System.Data.IDbTransaction transaction)
{
if (m_blockListHashLookup != null)
{
bool b;
if (m_blockListHashLookup.TryGet(hash, -1, out b))
{
return;
}
}
else
{
m_findblocklisthashCommand.Transaction = transaction;
m_findblocklisthashCommand.SetParameterValue(0, hash);
var r = m_findblocklisthashCommand.ExecuteScalar();
if (r != null && r != DBNull.Value)
{
return;
}
}
if (m_blockListHashLookup != null)
{
m_blockListHashLookup.Add(hash, -1, false);
}
m_insertBlockset.Transaction = transaction;
m_insertBlockset.SetParameterValue(0, hash);
var index = 0L;
foreach (var s in blocklisthashes)
{
m_insertBlockset.SetParameterValue(1, s);
m_insertBlockset.SetParameterValue(2, index++);
m_insertBlockset.ExecuteNonQuery();
}
}
public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
{
AddEntry(FilelistEntryType.File, filesetid, path, time, blocksetid, metahash, metahashsize, transaction);
}
protected virtual void Dispose(bool disposing)
{
if(!isDisposed)
{
if (disposing)
{
if (_trx != null) { _trx.Dispose(); _trx = null; }
}
}
// Code to dispose the unmanaged resources
// held by the class
_trx = null;
_conn = null;
isDisposed = true;
}
internal DBTransaction(System.Data.IDbTransaction Transaction, DBConnection Connection)
{
_trx = Transaction;
_conn = Connection;
}
public void Run(string[] sources, Library.Utility.IFilter filter)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);
using(m_database = new LocalBackupDatabase(m_options.Dbpath, m_options))
{
m_result.SetDatabase(m_database);
m_result.Dryrun = m_options.Dryrun;
Utility.UpdateOptionsFromDb(m_database, m_options);
Utility.VerifyParameters(m_database, m_options);
if (m_database.RepairInProgress)
throw new Exception("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.");
m_blocksize = m_options.Blocksize;
m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)];
m_blocklistbuffer = new byte[m_options.Blocksize];
m_blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
m_filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
if (m_blockhasher == null)
throw new Exception(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
if (m_filehasher == null)
throw new Exception(Strings.Common.InvalidHashAlgorithm(m_options.FileHashAlgorithm));
if (!m_blockhasher.CanReuseTransform)
throw new Exception(Strings.Common.InvalidCryptoSystem(m_options.BlockHashAlgorithm));
if (!m_filehasher.CanReuseTransform)
throw new Exception(Strings.Common.InvalidCryptoSystem(m_options.FileHashAlgorithm));
m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
// If there is no filter, we set an empty filter to simplify the code
// If there is a filter, we make sure that the sources are included
m_filter = filter ?? new Library.Utility.FilterExpression();
m_sourceFilter = new Library.Utility.FilterExpression(sources, true);
m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN);
System.Threading.Thread parallelScanner = null;
try
{
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
using(var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
{
using(var snapshot = GetSnapshot(sources, m_options, m_result))
{
// Start parallel scan
if (m_options.ChangedFilelist == null || m_options.ChangedFilelist.Length < 1)
{
parallelScanner = new System.Threading.Thread(CountFilesThread) {
Name = "Read ahead file counter",
IsBackground = true
};
parallelScanner.Start(snapshot);
}
PreBackupVerify(backend);
// Verify before uploading a synthetic list
m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
UploadSyntheticFilelist(backend);
m_database.BuildLookupTable(m_options);
m_transaction = m_database.BeginTransaction();
var repcnt = 0;
while(repcnt < 100 && m_database.GetRemoteVolumeID(filesetvolume.RemoteFilename) >= 0)
filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++));
if (m_database.GetRemoteVolumeID(filesetvolume.RemoteFilename) >= 0)
throw new Exception("Unable to generate a unique fileset name");
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
var filesetvolumeid = m_database.RegisterRemoteVolume(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);
m_database.CreateFileset(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time, m_transaction);
RunMainOperation(snapshot, backend);
//If the scanner is still running for some reason, make sure we kill it now
if (parallelScanner != null && parallelScanner.IsAlive)
parallelScanner.Abort();
}
var lastVolumeSize = FinalizeRemoteVolumes(backend);
using(new Logging.Timer("UpdateChangeStatistics"))
m_database.UpdateChangeStatistics(m_result);
using(new Logging.Timer("VerifyConsistency"))
m_database.VerifyConsistency(m_transaction, m_options.Blocksize, m_options.BlockhashSize);
UploadRealFileList(backend, filesetvolume);
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
using(new Logging.Timer("Async backend wait"))
backend.WaitForComplete(m_database, m_transaction);
if (m_result.TaskControlRendevouz() != TaskControlState.Stop)
CompactIfRequired(backend, lastVolumeSize);
if (m_options.UploadVerificationFile)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
FilelistProcessor.UploadVerificationFile(backend.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
}
if (m_options.Dryrun)
{
m_transaction.Rollback();
m_transaction = null;
}
else
{
using(new Logging.Timer("CommitFinalizingBackup"))
m_transaction.Commit();
m_transaction = null;
m_database.Vacuum();
if (m_result.TaskControlRendevouz() != TaskControlState.Stop && !m_options.NoBackendverification)
{
PostBackupVerification();
}
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
m_database.WriteResults();
m_database.PurgeLogData(m_options.LogRetention);
return;
}
}
catch (Exception ex)
{
m_result.AddError("Fatal error", ex);
throw;
}
finally
{
if (parallelScanner != null && parallelScanner.IsAlive)
{
parallelScanner.Abort();
parallelScanner.Join(500);
if (parallelScanner.IsAlive)
m_result.AddWarning("Failed to terminate filecounter thread", null);
}
if (m_transaction != null)
try { m_transaction.Rollback(); }
catch (Exception ex) { m_result.AddError(string.Format("Rollback error: {0}", ex.Message), ex); }
}
}
}
private void UploadRealFileList(BackendManager backend, FilesetVolumeWriter filesetvolume)
{
var changeCount =
m_result.AddedFiles + m_result.ModifiedFiles + m_result.DeletedFiles +
m_result.AddedFolders + m_result.ModifiedFolders + m_result.DeletedFolders +
m_result.AddedSymlinks + m_result.ModifiedSymlinks + m_result.DeletedSymlinks;
//Changes in the filelist triggers a filelist upload
if (m_options.UploadUnchangedBackups || changeCount > 0)
{
using(new Logging.Timer("Uploading a new fileset"))
{
if (!string.IsNullOrEmpty(m_options.ControlFiles))
foreach(var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
filesetvolume.AddControlFile(p, m_options.GetCompressionHintFromFilename(p));
m_database.WriteFileset(filesetvolume, m_transaction);
filesetvolume.Close();
if (m_options.Dryrun)
m_result.AddDryrunMessage(string.Format("Would upload fileset volume: {0}, size: {1}", filesetvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(filesetvolume.LocalFilename).Length)));
else
{
m_database.UpdateRemoteVolume(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
using(new Logging.Timer("CommitUpdateRemoteVolume"))
m_transaction.Commit();
m_transaction = m_database.BeginTransaction();
backend.Put(filesetvolume);
using(new Logging.Timer("CommitUpdateRemoteVolume"))
m_transaction.Commit();
m_transaction = m_database.BeginTransaction();
}
}
}
else
{
m_result.AddVerboseMessage("removing temp files, as no data needs to be uploaded");
m_database.RemoveRemoteVolume(filesetvolume.RemoteFilename, m_transaction);
}
}
private long FinalizeRemoteVolumes(BackendManager backend)
{
var lastVolumeSize = -1L;
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Finalize);
using(new Logging.Timer("FinalizeRemoteVolumes"))
{
if (m_blockvolume != null && m_blockvolume.SourceSize > 0)
{
lastVolumeSize = m_blockvolume.SourceSize;
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("Would upload block volume: {0}, size: {1}", m_blockvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_blockvolume.LocalFilename).Length)));
if (m_indexvolume != null)
{
m_blockvolume.Close();
UpdateIndexVolume();
m_indexvolume.FinishVolume(Library.Utility.Utility.CalculateHash(m_blockvolume.LocalFilename), new FileInfo(m_blockvolume.LocalFilename).Length);
m_result.AddDryrunMessage(string.Format("Would upload index volume: {0}, size: {1}", m_indexvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_indexvolume.LocalFilename).Length)));
}
m_blockvolume.Dispose();
m_blockvolume = null;
m_indexvolume.Dispose();
m_indexvolume = null;
}
else
{
m_database.UpdateRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
m_blockvolume.Close();
UpdateIndexVolume();
using(new Logging.Timer("CommitUpdateRemoteVolume"))
m_transaction.Commit();
m_transaction = m_database.BeginTransaction();
backend.Put(m_blockvolume, m_indexvolume);
using(new Logging.Timer("CommitUpdateRemoteVolume"))
m_transaction.Commit();
m_transaction = m_database.BeginTransaction();
m_blockvolume = null;
m_indexvolume = null;
}
}
}
return lastVolumeSize;
}
public void StartUpdate(CacheStore store)
{
m_trans = store.StartUpdate();
}
public bool UpdateBlockset(string hash, IEnumerable <string> blocklisthashes, System.Data.IDbTransaction transaction)
{
m_findblocklisthashCommand.Transaction = transaction;
m_findblocklisthashCommand.SetParameterValue(0, hash);
var r = m_findblocklisthashCommand.ExecuteScalar();
if (r != null && r != DBNull.Value)
{
return(false);
}
m_insertBlockset.Transaction = transaction;
m_insertBlockset.SetParameterValue(0, hash);
var index = 0L;
foreach (var s in blocklisthashes)
{
m_insertBlockset.SetParameterValue(1, s);
m_insertBlockset.SetParameterValue(2, index++);
m_insertBlockset.ExecuteNonQuery();
}
return(true);
}
public void AddDirectoryEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
{
AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metahash, metahashsize, transaction);
}
public Basiclist(System.Data.IDbConnection connection, string volumename)
{
m_connection = connection;
m_volumename = volumename;
m_transaction = m_connection.BeginTransaction();
var tablename = TABLE_PREFIX + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using(var cmd = m_connection.CreateCommand())
{
cmd.Transaction = m_transaction;
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" {1}", tablename, TABLEFORMAT));
m_tablename = tablename;
}
m_insertCommand = m_connection.CreateCommand();
m_insertCommand.Transaction = m_transaction;
m_insertCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" {1}", m_tablename, INSERTCOMMAND);
m_insertCommand.AddParameters(INSERTARGUMENTS);
}
public long AddBlockset(string fullhash, long size, IEnumerable <string> blocklisthashes, System.Data.IDbTransaction transaction)
{
var blocksetid = -1L;
if (m_fileHashLookup != null)
{
if (m_fileHashLookup.TryGet(fullhash, size, out blocksetid))
{
return(blocksetid);
}
else
{
blocksetid = -1;
}
}
else
{
m_findBlocksetCommand.Transaction = transaction;
m_findBlocksetCommand.SetParameterValue(0, size);
m_findBlocksetCommand.SetParameterValue(1, fullhash);
var r = m_findBlocksetCommand.ExecuteScalar();
if (r != null && r != DBNull.Value)
{
return(Convert.ToInt64(r));
}
}
m_insertBlocksetCommand.Transaction = transaction;
m_insertBlocksetCommand.SetParameterValue(0, size);
m_insertBlocksetCommand.SetParameterValue(1, fullhash);
blocksetid = Convert.ToInt64(m_insertBlocksetCommand.ExecuteScalar());
if (m_fileHashLookup != null)
{
m_fileHashLookup.Add(fullhash, size, blocksetid);
}
if (blocklisthashes != null)
{
var index = 0L;
m_insertBlocklistHashCommand.Transaction = transaction;
m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid);
foreach (var hash in blocklisthashes)
{
if (!string.IsNullOrEmpty(hash))
{
m_insertBlocklistHashCommand.SetParameterValue(1, index++);
m_insertBlocklistHashCommand.SetParameterValue(2, hash);
m_insertBlocklistHashCommand.ExecuteNonQuery();
}
}
}
return(blocksetid);
}
public virtual void Dispose()
{
if (m_tablename != null)
try
{
using(var cmd = m_connection.CreateCommand())
{
cmd.Transaction = m_transaction;
cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tablename));
}
}
catch {}
finally { m_tablename = null; }
if (m_insertCommand != null)
try { m_insertCommand.Dispose(); }
catch {}
finally { m_insertCommand = null; }
if (m_transaction != null)
try { m_transaction.Rollback(); }
catch {}
finally { m_transaction = null; }
}
public System.Data.IDbCommand CreateCommand(System.Data.IDbConnection _Connection, System.Data.IDbTransaction _Transaction, string _SQL)
{
return(new OracleCommand(_SQL, (OracleConnection)_Connection, (OracleTransaction)_Transaction));
}
public void Run(string[] sources, Library.Utility.IFilter filter)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);
using(m_database = new LocalBackupDatabase(m_options.Dbpath, m_options))
{
m_result.SetDatabase(m_database);
m_result.Dryrun = m_options.Dryrun;
Utility.VerifyParameters(m_database, m_options);
m_database.VerifyConsistency(null);
// If there is no filter, we set an empty filter to simplify the code
// If there is a filter, we make sure that the sources are included
m_filter = filter ?? new Library.Utility.FilterExpression();
m_sourceFilter = new Library.Utility.FilterExpression(sources, true);
var lastVolumeSize = -1L;
m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN);
System.Threading.Thread parallelScanner = null;
try
{
m_snapshot = GetSnapshot(sources, m_options, m_result);
// Start parallel scan
if (m_options.ChangedFilelist == null || m_options.ChangedFilelist.Length < 1)
{
parallelScanner = new System.Threading.Thread(CountFilesThread) {
Name = "Read ahead file counter",
IsBackground = true
};
parallelScanner.Start();
}
using(m_backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
using(m_filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
{
var incompleteFilesets = m_database.GetIncompleteFilesets(null).OrderBy(x => x.Value).ToArray();
if (incompleteFilesets.Length != 0)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize);
m_result.AddMessage(string.Format("Uploading filelist from previous interrupted backup"));
using(var trn = m_database.BeginTransaction())
{
var incompleteSet = incompleteFilesets.Last();
var badIds = from n in incompleteFilesets select n.Key;
var prevs = (from n in m_database.FilesetTimes
where
n.Key < incompleteSet.Key
&&
!badIds.Contains(n.Key)
orderby n.Key
select n.Key).ToArray();
var prevId = prevs.Length == 0 ? -1 : prevs.Last();
FilesetVolumeWriter fsw = null;
try
{
var s = 1;
var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s);
var oldFilesetID = incompleteSet.Key;
// Probe for an unused filename
while (s < 60)
{
var id = m_database.GetRemoteVolumeID(VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, fileTime));
if (id < 0)
break;
fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s);
}
fsw = new FilesetVolumeWriter(m_options, fileTime);
fsw.VolumeID = m_database.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);
var newFilesetID = m_database.CreateFileset(fsw.VolumeID, fileTime, trn);
m_database.LinkFilesetToVolume(newFilesetID, fsw.VolumeID, trn);
m_database.AppendFilesFromPreviousSet(trn, null, newFilesetID, prevId, fileTime);
m_database.WriteFileset(fsw, trn, newFilesetID);
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("Would upload fileset: {0}, size: {1}", fsw.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(fsw.LocalFilename).Length)));
}
else
{
m_database.UpdateRemoteVolume(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null, trn);
using(new Logging.Timer("CommitUpdateFilelistVolume"))
trn.Commit();
m_backend.Put(fsw);
fsw = null;
}
}
finally
{
if (fsw != null)
try { fsw.Dispose(); }
catch { fsw = null; }
}
}
}
if (!m_options.NoBackendverification)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupVerify);
using(new Logging.Timer("PreBackupVerify"))
{
try
{
FilelistProcessor.VerifyRemoteList(m_backend, m_options, m_database, m_result.BackendWriter);
}
catch (Exception ex)
{
if (m_options.AutoCleanup)
{
m_result.AddWarning("Backend verification failed, attempting automatic cleanup", ex);
m_result.RepairResults = new RepairResults(m_result);
new RepairHandler(m_backend.BackendUrl, m_options, (RepairResults)m_result.RepairResults).Run();
m_result.AddMessage("Backend cleanup finished, retrying verification");
FilelistProcessor.VerifyRemoteList(m_backend, m_options, m_database, m_result.BackendWriter);
}
else
throw;
}
}
}
m_database.BuildLookupTable(m_options);
m_transaction = m_database.BeginTransaction();
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
var filesetvolumeid = m_database.RegisterRemoteVolume(m_filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);
m_database.CreateFileset(filesetvolumeid, VolumeBase.ParseFilename(m_filesetvolume.RemoteFilename).Time, m_transaction);
m_blockvolume = new BlockVolumeWriter(m_options);
m_blockvolume.VolumeID = m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_transaction);
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
m_indexvolume = new IndexVolumeWriter(m_options);
m_indexvolume.VolumeID = m_database.RegisterRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_transaction);
}
var filterhandler = new FilterHandler(m_snapshot, m_attributeFilter, m_sourceFilter, m_filter, m_symlinkPolicy, m_options.HardlinkPolicy, m_result);
using(new Logging.Timer("BackupMainOperation"))
{
if (m_options.ChangedFilelist != null && m_options.ChangedFilelist.Length >= 1)
{
m_result.AddVerboseMessage("Processing supplied change list instead of enumerating filesystem");
m_result.OperationProgressUpdater.UpdatefileCount(m_options.ChangedFilelist.Length, 0, true);
foreach(var p in m_options.ChangedFilelist)
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
m_result.AddMessage("Stopping backup operation on request");
break;
}
FileAttributes fa = new FileAttributes();
try
{
fa = m_snapshot.GetAttributes(p);
}
catch (Exception ex)
{
m_result.AddWarning(string.Format("Failed to read attributes: {0}, message: {1}", p, ex.Message), ex);
}
if (filterhandler.AttributeFilter(null, p, fa))
{
try
{
this.HandleFilesystemEntry(p, fa);
}
catch (Exception ex)
{
m_result.AddWarning(string.Format("Failed to process element: {0}, message: {1}", p, ex.Message), ex);
}
}
}
m_database.AppendFilesFromPreviousSet(m_transaction, m_options.DeletedFilelist);
}
else
{
foreach(var path in m_snapshot.EnumerateFilesAndFolders(filterhandler.AttributeFilter))
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
m_result.AddMessage("Stopping backup operation on request");
break;
}
this.HandleFilesystemEntry(path, m_snapshot.GetAttributes(path));
}
}
//If the scanner is still running for some reason, make sure we kill it now
if (parallelScanner != null && parallelScanner.IsAlive)
parallelScanner.Abort();
// We no longer need to snapshot active
try { m_snapshot.Dispose(); }
finally { m_snapshot = null; }
m_result.OperationProgressUpdater.UpdatefileCount(m_result.ExaminedFiles, m_result.SizeOfExaminedFiles, true);
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Finalize);
using(new Logging.Timer("FinalizeRemoteVolumes"))
{
if (m_blockvolume.SourceSize > 0)
{
lastVolumeSize = m_blockvolume.SourceSize;
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("Would upload block volume: {0}, size: {1}", m_blockvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_blockvolume.LocalFilename).Length)));
if (m_indexvolume != null)
{
m_blockvolume.Close();
UpdateIndexVolume();
m_indexvolume.FinishVolume(Library.Utility.Utility.CalculateHash(m_blockvolume.LocalFilename), new FileInfo(m_blockvolume.LocalFilename).Length);
m_result.AddDryrunMessage(string.Format("Would upload index volume: {0}, size: {1}", m_indexvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_indexvolume.LocalFilename).Length)));
}
m_blockvolume.Dispose();
m_blockvolume = null;
m_indexvolume.Dispose();
m_indexvolume = null;
}
else
{
m_database.UpdateRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
m_blockvolume.Close();
UpdateIndexVolume();
using(new Logging.Timer("CommitUpdateRemoteVolume"))
m_transaction.Commit();
m_transaction = m_database.BeginTransaction();
m_backend.Put(m_blockvolume, m_indexvolume);
m_blockvolume = null;
m_indexvolume = null;
}
}
else
{
m_database.RemoveRemoteVolume(m_blockvolume.RemoteFilename, m_transaction);
if (m_indexvolume != null)
m_database.RemoveRemoteVolume(m_indexvolume.RemoteFilename, m_transaction);
}
}
using(new Logging.Timer("UpdateChangeStatistics"))
m_database.UpdateChangeStatistics(m_result);
using(new Logging.Timer("VerifyConsistency"))
m_database.VerifyConsistency(m_transaction);
var changeCount =
m_result.AddedFiles + m_result.ModifiedFiles + m_result.DeletedFiles +
m_result.AddedFolders + m_result.ModifiedFolders + m_result.DeletedFolders +
m_result.AddedSymlinks + m_result.ModifiedSymlinks + m_result.DeletedSymlinks;
//Changes in the filelist triggers a filelist upload
if (m_options.UploadUnchangedBackups || changeCount > 0)
{
using(new Logging.Timer("Uploading a new fileset"))
{
if (!string.IsNullOrEmpty(m_options.ControlFiles))
foreach(var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
m_filesetvolume.AddControlFile(p, m_options.GetCompressionHintFromFilename(p));
m_database.WriteFileset(m_filesetvolume, m_transaction);
m_filesetvolume.Close();
if (m_options.Dryrun)
m_result.AddDryrunMessage(string.Format("Would upload fileset volume: {0}, size: {1}", m_filesetvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_filesetvolume.LocalFilename).Length)));
else
{
m_database.UpdateRemoteVolume(m_filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
using(new Logging.Timer("CommitUpdateRemoteVolume"))
m_transaction.Commit();
m_transaction = m_database.BeginTransaction();
m_backend.Put(m_filesetvolume);
}
}
}
else
{
m_result.AddVerboseMessage("removing temp files, as no data needs to be uploaded");
m_database.RemoveRemoteVolume(m_filesetvolume.RemoteFilename, m_transaction);
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
using(new Logging.Timer("Async backend wait"))
m_backend.WaitForComplete(m_database, m_transaction);
if (m_result.TaskControlRendevouz() != TaskControlState.Stop)
{
if (m_options.KeepTime.Ticks > 0 || m_options.KeepVersions != 0)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete);
m_result.DeleteResults = new DeleteResults(m_result);
using(var db = new LocalDeleteDatabase(m_database))
new DeleteHandler(m_backend.BackendUrl, m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, m_transaction, true, lastVolumeSize <= m_options.SmallFileSize);
}
else if (lastVolumeSize <= m_options.SmallFileSize && !m_options.NoAutoCompact)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact);
m_result.CompactResults = new CompactResults(m_result);
using(var db = new LocalDeleteDatabase(m_database))
new CompactHandler(m_backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, m_transaction);
}
}
if (m_options.UploadVerificationFile)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
FilelistProcessor.UploadVerificationFile(m_backend.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
}
if (m_options.Dryrun)
{
m_transaction.Rollback();
m_transaction = null;
}
else
{
using(new Logging.Timer("CommitFinalizingBackup"))
m_transaction.Commit();
m_transaction = null;
m_database.Vacuum();
if (m_result.TaskControlRendevouz() != TaskControlState.Stop && !m_options.NoBackendverification)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify);
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
{
using(new Logging.Timer("AfterBackupVerify"))
FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter);
backend.WaitForComplete(m_database, null);
}
if (m_options.BackupTestSampleCount > 0 && m_database.GetRemoteVolumes().Count() > 0)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest);
m_result.TestResults = new TestResults(m_result);
using(var testdb = new LocalTestDatabase(m_database))
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, testdb))
new TestHandler(m_backendurl, m_options, new TestResults(m_result))
.DoRun(m_options.BackupTestSampleCount, testdb, backend);
}
}
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
m_database.WriteResults();
return;
}
}
catch (Exception ex)
{
m_result.AddError("Fatal error", ex);
throw;
}
finally
{
if (parallelScanner != null && parallelScanner.IsAlive)
{
parallelScanner.Abort();
parallelScanner.Join(500);
if (parallelScanner.IsAlive)
m_result.AddWarning("Failed to terminate filecounter thread", null);
}
if (m_snapshot != null)
try { m_snapshot.Dispose(); }
catch (Exception ex) { m_result.AddError(string.Format("Failed to dispose snapshot"), ex); }
finally { m_snapshot = null; }
if (m_transaction != null)
try { m_transaction.Rollback(); }
catch (Exception ex) { m_result.AddError(string.Format("Rollback error: {0}", ex.Message), ex); }
}
}
}
public bool Save(iSqlTransaction trans, int cartid, List<CartitemInfo> list)
{
try
{
try
{
foreach (CartitemInfo info in list)
{
info.CartID = cartid;
this.Saveitem(trans, info);
}
//trans.Commit();
}
catch (Exception ex)
{
trans.Rollback();
throw ex;
}
return true;
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// Adds the found file data to the output unless the block already exists
/// </summary>
/// <param name="key">The block hash</param>
/// <param name="data">The data matching the hash</param>
/// <param name="len">The size of the data</param>
/// <param name="offset">The offset into the data</param>
/// <param name="hint">Hint for compression module</param>
/// <param name="isBlocklistData">Indicates if the block is list data</param>
private bool AddBlockToOutput(string key, byte[] data, int offset, int len, CompressionHint hint, bool isBlocklistData)
{
if (m_database.AddBlock(key, len, m_blockvolume.VolumeID, m_transaction))
{
m_blockvolume.AddBlock(key, data, offset, len, hint);
//TODO: In theory a normal data block and blocklist block could be equal.
// this would cause the index file to not contain all data,
// if the data file is added before the blocklist data
// ... highly theoretical ...
if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full && isBlocklistData)
m_indexvolume.WriteBlocklist(key, data, offset, len);
if (m_blockvolume.Filesize > m_options.VolumeSize - m_options.Blocksize)
{
if (m_options.Dryrun)
{
m_blockvolume.Close();
m_result.AddDryrunMessage(string.Format("Would upload block volume: {0}, size: {1}", m_blockvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_blockvolume.LocalFilename).Length)));
if (m_indexvolume != null)
{
UpdateIndexVolume();
m_indexvolume.FinishVolume(Library.Utility.Utility.CalculateHash(m_blockvolume.LocalFilename), new FileInfo(m_blockvolume.LocalFilename).Length);
m_result.AddDryrunMessage(string.Format("Would upload index volume: {0}, size: {1}", m_indexvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_indexvolume.LocalFilename).Length)));
m_indexvolume.Dispose();
m_indexvolume = null;
}
m_blockvolume.Dispose();
m_blockvolume = null;
m_indexvolume.Dispose();
m_indexvolume = null;
}
else
{
//When uploading a new volume, we register the volumes and then flush the transaction
// this ensures that the local database and remote storage are as closely related as possible
m_database.UpdateRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
m_blockvolume.Close();
UpdateIndexVolume();
m_backend.FlushDbMessages(m_database, m_transaction);
m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN);
using(new Logging.Timer("CommitAddBlockToOutputFlush"))
m_transaction.Commit();
m_transaction = m_database.BeginTransaction();
m_backend.Put(m_blockvolume, m_indexvolume);
m_blockvolume = null;
m_indexvolume = null;
}
m_blockvolume = new BlockVolumeWriter(m_options);
m_blockvolume.VolumeID = m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_transaction);
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
m_indexvolume = new IndexVolumeWriter(m_options);
m_indexvolume.VolumeID = m_database.RegisterRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_transaction);
}
}
return true;
}
return false;
}
private bool Deleteitem(iSqlTransaction trans, int iid, int belongto)
{
try
{
if (trans == null) return false;
string SQL = "DELETE FROM " + TABLENAME + " WHERE iid=" + iid + " AND belongto=" + belongto;
HELPER.executeNonQuery(trans, SQL);
return true;
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// Saves the current record to the database.
/// </summary>
/// <param name="tx">Optional transaction</param>
/// <returns>Wether the operation was successful</returns>
public override bool Save(System.Data.IDbTransaction tx = null)
{
return(Save(tx, true));
}
private bool Deleteitem(iSqlTransaction trans)
{
try
{
if (trans == null) return false;
string SQL = "DELETE FROM " + TABLENAME;
HELPER.executeNonQuery(trans, SQL);
return true;
}
catch (Exception ex)
{
throw ex;
}
}
public Task GetNextTask(string component)
{
if (Workspace == null)
{
return(null);
}
if (this.DataBase.State != System.Data.ConnectionState.Open)
{
this.DataBase.Open();
}
qGen.Where WhereEstacion = new qGen.Where(qGen.AndOr.Or);
WhereEstacion.AddWithValue("estacion", this.DataBase.EscapeString(Lfx.Environment.SystemInformation.MachineName));
WhereEstacion.AddWithValue("estacion", "*");
qGen.Where WhereFecha = new qGen.Where(qGen.AndOr.Or);
WhereFecha.AddWithValue("fechaejecutar", qGen.ComparisonOperators.LessOrEqual, new qGen.SqlExpression("NOW()"));
WhereFecha.AddWithValue("fechaejecutar", null);
m_LastGetTask = DateTime.Now;
qGen.Select NextTask = new qGen.Select("sys_programador");
NextTask.WhereClause = new qGen.Where("estado", 0);
NextTask.WhereClause.AddWithValue("componente", component);
NextTask.WhereClause.AddWithValue(WhereEstacion);
NextTask.WhereClause.AddWithValue(WhereFecha);
NextTask.Order = "id_evento";
Lfx.Data.Row TaskRow;
try {
TaskRow = this.DataBase.FirstRowFromSelect(NextTask);
} catch {
TaskRow = null;
}
if (TaskRow != null)
{
Task Result = new Task();
Result.Id = System.Convert.ToInt32(TaskRow["id_evento"]);
Result.Command = TaskRow["comando"].ToString();
Result.Component = TaskRow["componente"].ToString();
Result.Creator = TaskRow["crea_usuario"].ToString();
Result.CreatorComputerName = TaskRow["crea_estacion"].ToString();
Result.ComputerName = TaskRow["estacion"].ToString();
Result.Schedule = System.Convert.ToDateTime(TaskRow["fecha"]);
Result.Status = System.Convert.ToInt32(TaskRow["estado"]);
//Elimino tareas viejas
qGen.Update Actualizar = new qGen.Update("sys_programador", new qGen.Where("id_evento", Result.Id));
Actualizar.ColumnValues.AddWithValue("estado", 1);
using (System.Data.IDbTransaction Trans = this.DataBase.BeginTransaction()) {
this.DataBase.ExecuteNonQuery(Actualizar);
this.DataBase.ExecuteNonQuery(new qGen.Delete("sys_programador", new qGen.Where("fecha", qGen.ComparisonOperators.LessThan, System.DateTime.Now.AddDays(-7))));
Trans.Commit();
}
return(Result);
}
else
{
return(null);
}
}
public long AddMetadataset(string metahash, long metahashsize, IEnumerable <string> metablocklisthashes, long expectedmetablocklisthashes, System.Data.IDbTransaction transaction)
{
var metadataid = -1L;
if (metahash == null)
{
return(metadataid);
}
m_findMetadatasetCommand.Transaction = transaction;
m_findMetadatasetCommand.SetParameterValue(0, metahash);
m_findMetadatasetCommand.SetParameterValue(1, metahashsize);
metadataid = m_findMetadatasetCommand.ExecuteScalarInt64(-1);
if (metadataid != -1)
{
return(metadataid);
}
var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction);
m_insertMetadatasetCommand.Transaction = transaction;
m_insertMetadatasetCommand.SetParameterValue(0, blocksetid);
metadataid = m_insertMetadatasetCommand.ExecuteScalarInt64(-1);
return(metadataid);
}
private IEnumerable <KeyValuePair <string, long> > DoDelete(LocalDeleteDatabase db, BackendManager backend, IEnumerable <IRemoteVolume> deleteableVolumes, ref System.Data.IDbTransaction transaction)
{
// Mark all volumes as disposable
foreach (var f in deleteableVolumes)
{
db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction);
}
// Before we commit the current state, make sure the backend has caught up
backend.WaitForEmpty(db, transaction);
if (!m_options.Dryrun)
{
transaction.Commit();
transaction = db.BeginTransaction();
}
return(PerformDelete(backend, db.GetDeletableVolumes(deleteableVolumes, transaction)));
}
public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, System.Data.IDbTransaction transaction)
{
m_insertSmallBlockset.Transaction = transaction;
m_insertSmallBlockset.SetParameterValue(0, filehash);
m_insertSmallBlockset.SetParameterValue(1, blockhash);
m_insertSmallBlockset.SetParameterValue(2, blocksize);
m_insertSmallBlockset.ExecuteNonQuery();
}
internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction, BackendManager sharedBackend)
{
var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction);
report.ReportCompactData();
if (report.ShouldReclaim || report.ShouldCompact)
{
// Workaround where we allow a running backendmanager to be used
using (var bk = sharedBackend == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null)
{
var backend = bk ?? sharedBackend;
if (!hasVerifiedBackend && !m_options.NoBackendverification)
{
FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
}
BlockVolumeWriter newvol = new BlockVolumeWriter(m_options);
newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
IndexVolumeWriter newvolindex = null;
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
newvolindex = new IndexVolumeWriter(m_options);
newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
}
long blocksInVolume = 0;
byte[] buffer = new byte[m_options.Blocksize];
var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
//These are for bookkeeping
var uploadedVolumes = new List <KeyValuePair <string, long> >();
var deletedVolumes = new List <KeyValuePair <string, long> >();
var downloadedVolumes = new List <KeyValuePair <string, long> >();
//We start by deleting unused volumes to save space before uploading new stuff
var fullyDeleteable = (from v in remoteList
where report.DeleteableVolumes.Contains(v.Name)
select(IRemoteVolume) v).ToList();
deletedVolumes.AddRange(DoDelete(db, backend, fullyDeleteable, ref transaction));
// This list is used to pick up unused volumes,
// so they can be deleted once the upload of the
// required fragments is complete
var deleteableVolumes = new List <IRemoteVolume>();
if (report.ShouldCompact)
{
newvolindex?.StartVolume(newvol.RemoteFilename);
var volumesToDownload = (from v in remoteList
where report.CompactableVolumes.Contains(v.Name)
select(IRemoteVolume) v).ToList();
using (var q = db.CreateBlockQueryHelper(transaction))
{
foreach (var entry in new AsyncDownloader(volumesToDownload, backend))
{
using (var tmpfile = entry.TempFile)
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, transaction);
return(false);
}
downloadedVolumes.Add(new KeyValuePair <string, long>(entry.Name, entry.Size));
var inst = VolumeBase.ParseFilename(entry.Name);
using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
{
foreach (var e in f.Blocks)
{
if (q.UseBlock(e.Key, e.Value, transaction))
{
//TODO: How do we get the compression hint? Reverse query for filename in db?
var s = f.ReadBlock(e.Key, buffer);
if (s != e.Value)
{
throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value));
}
newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible);
if (newvolindex != null)
{
newvolindex.AddBlock(e.Key, e.Value);
}
db.MoveBlockToNewVolume(e.Key, e.Value, newvol.VolumeID, transaction);
blocksInVolume++;
if (newvol.Filesize > m_options.VolumeSize)
{
uploadedVolumes.Add(new KeyValuePair <string, long>(newvol.RemoteFilename, newvol.Filesize));
if (newvolindex != null)
{
uploadedVolumes.Add(new KeyValuePair <string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
}
if (!m_options.Dryrun)
{
backend.Put(newvol, newvolindex);
}
else
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadGeneratedBlockset", "Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize));
}
newvol = new BlockVolumeWriter(m_options);
newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
newvolindex = new IndexVolumeWriter(m_options);
newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
newvolindex.StartVolume(newvol.RemoteFilename);
}
blocksInVolume = 0;
//After we upload this volume, we can delete all previous encountered volumes
deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction));
deleteableVolumes = new List <IRemoteVolume>();
}
}
}
}
deleteableVolumes.Add(entry);
}
}
if (blocksInVolume > 0)
{
uploadedVolumes.Add(new KeyValuePair <string, long>(newvol.RemoteFilename, newvol.Filesize));
if (newvolindex != null)
{
uploadedVolumes.Add(new KeyValuePair <string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
}
if (!m_options.Dryrun)
{
backend.Put(newvol, newvolindex);
}
else
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadGeneratedBlockset", "Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize));
}
}
else
{
db.RemoveRemoteVolume(newvol.RemoteFilename, transaction);
if (newvolindex != null)
{
db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction);
newvolindex.FinishVolume(null, 0);
}
}
}
}
else
{
newvolindex?.Dispose();
newvol.Dispose();
}
deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction));
var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
m_result.DeletedFileCount = deletedVolumes.Count;
m_result.DownloadedFileCount = downloadedVolumes.Count;
m_result.UploadedFileCount = uploadedVolumes.Count;
m_result.DeletedFileSize = deletedSize;
m_result.DownloadedFileSize = downloadSize;
m_result.UploadedFileSize = uploadSize;
m_result.Dryrun = m_options.Dryrun;
if (m_result.Dryrun)
{
if (downloadedVolumes.Count == 0)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would delete {0} files, which would reduce storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize));
}
else
{
Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}",
m_result.DownloadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DownloadedFileSize),
m_result.DeletedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), m_result.UploadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize),
m_result.DeletedFileCount - m_result.UploadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize));
}
}
else
{
if (m_result.DownloadedFileCount == 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Deleted {0} files, which reduced storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize));
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}",
m_result.DownloadedFileCount,
Library.Utility.Utility.FormatSizeString(downloadSize),
m_result.DeletedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize),
m_result.UploadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize),
m_result.DeletedFileCount - m_result.UploadedFileCount,
Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize));
}
}
backend.WaitForComplete(db, transaction);
}
m_result.EndTime = DateTime.UtcNow;
return((m_result.DeletedFileCount + m_result.UploadedFileCount) > 0);
}
else
{
m_result.EndTime = DateTime.UtcNow;
return(false);
}
}
public void FindMissingBlocklistHashes(long hashsize, System.Data.IDbTransaction transaction)
{
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = transaction;
//Update all small blocklists and matching blocks
var selectSmallBlocks = string.Format(@"SELECT ""Blockset"".""Fullhash"", ""Blockset"".""Length"" FROM ""Blockset"" WHERE ""Blockset"".""Length"" <= {0}", m_blocksize);
var selectBlockHashes = string.Format(
@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " +
SELECT_BLOCKLIST_ENTRIES +
@" )",
m_blocksize,
hashsize,
m_tempblocklist
);
var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )";
var selectNewBlocks = string.Format(
@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" " +
@" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" +
@" LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" )" +
@" WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" ",
selectAllBlocks
);
var insertBlocksCommand =
@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " +
selectNewBlocks;
// Insert all known blocks into block table with volumeid = -1
cmd.ExecuteNonQuery(insertBlocksCommand);
// Update the cache with new blocks
if (m_blockHashLookup != null)
{
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = -1 "))
while (rd.Read())
{
var hash = rd.GetValue(0).ToString();
var size = Convert.ToInt64(rd.GetValue(1));
m_blockHashLookup.Add(hash, size, -1);
}
}
var selectBlocklistBlocksetEntries = string.Format(
@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " +
SELECT_BLOCKLIST_ENTRIES +
@") D, ""BlocklistHash"" E, ""Block"" F WHERE ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" ",
m_blocksize,
hashsize,
m_tempblocklist
);
var selectBlocksetEntries = string.Format(
@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" <= {0} ",
m_blocksize
);
var selectAllBlocksetEntries =
selectBlocklistBlocksetEntries +
@" UNION " +
selectBlocksetEntries;
var selectFiltered =
@"SELECT DISTINCT ""BlocksetID"", ""Index"", ""BlockID"" FROM (" +
selectAllBlocksetEntries +
@") A WHERE (""A"".""BlocksetID"" || ':' || ""A"".""Index"") NOT IN (SELECT (""BlocksetID"" || ':' || ""Index"") FROM ""BlocksetEntry"" )";
var insertBlocksetEntriesCommand =
@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered;
cmd.ExecuteNonQuery(insertBlocksetEntriesCommand);
}
}
public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, System.Data.IDbConnection connection, System.Data.IDbTransaction transaction)
{
this.ParentID = parentid;
m_parentdb = parentdb;
m_connection = connection;
m_transaction = transaction;
m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using (var cmd = m_connection.CreateCommand(m_transaction))
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER PRIMARY KEY) ", m_tablename));
}
public void AddSymlinkEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
{
AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metahash, metahashsize, transaction);
}
public ITemporaryFileset CreateTemporaryFileset(long parentid, System.Data.IDbTransaction transaction)
{
return(new TemporaryFileset(parentid, this, m_connection, transaction));
}
private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
{
var fileid = -1L;
var metadataid = AddMetadataset(metahash, metahashsize, transaction);
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
{
fileid = e.GetFilesetID(blocksetid, metadataid);
}
}
else
{
m_findFilesetCommand.Transaction = transaction;
m_findFilesetCommand.SetParameterValue(0, path);
m_findFilesetCommand.SetParameterValue(1, blocksetid);
m_findFilesetCommand.SetParameterValue(2, metadataid);
var r = m_findFilesetCommand.ExecuteScalar();
if (r != null && r != DBNull.Value)
{
fileid = Convert.ToInt64(r);
}
}
if (fileid < 0)
{
m_insertFileCommand.Transaction = transaction;
m_insertFileCommand.SetParameterValue(0, path);
m_insertFileCommand.SetParameterValue(1, blocksetid);
m_insertFileCommand.SetParameterValue(2, metadataid);
fileid = Convert.ToInt64(m_insertFileCommand.ExecuteScalar());
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
{
e.AddFilesetID(blocksetid, metadataid, fileid);
}
else
{
e = new PathEntryKeeper();
e.AddFilesetID(blocksetid, metadataid, fileid);
m_filesetLookup.Insert(path, e);
}
}
}
m_insertFilesetEntryCommand.Transaction = transaction;
m_insertFilesetEntryCommand.SetParameterValue(0, filesetid);
m_insertFilesetEntryCommand.SetParameterValue(1, fileid);
m_insertFilesetEntryCommand.SetParameterValue(2, NormalizeDateTimeToEpochSeconds(time));
m_insertFilesetEntryCommand.ExecuteNonQuery();
}
/// <summary>
/// Builds a lookup table to enable faster response to block queries
/// </summary>
/// <param name="volumename">The name of the volume to prepare for</param>
public IBlockQuery CreateBlockQueryHelper(Options options, System.Data.IDbTransaction transaction)
{
return(new BlockQuery(m_connection, options, transaction));
}
public void UpdateBlock(string hash, long size, long volumeID, System.Data.IDbTransaction transaction)
{
var currentVolumeId = -2L;
if (m_blockHashLookup != null)
{
if (!m_blockHashLookup.TryGet(hash, size, out currentVolumeId))
{
currentVolumeId = -2;
}
}
else
{
m_findHashBlockCommand.Transaction = transaction;
m_findHashBlockCommand.SetParameterValue(0, hash);
m_findHashBlockCommand.SetParameterValue(1, size);
var r = m_findHashBlockCommand.ExecuteScalar();
if (r != null && r != DBNull.Value)
{
currentVolumeId = Convert.ToInt64(r);
}
}
if (currentVolumeId == volumeID)
{
return;
}
if (currentVolumeId == -2)
{
//Insert
m_insertBlockCommand.Transaction = transaction;
m_insertBlockCommand.SetParameterValue(0, hash);
m_insertBlockCommand.SetParameterValue(1, size);
m_insertBlockCommand.SetParameterValue(2, volumeID);
m_insertBlockCommand.ExecuteNonQuery();
if (m_blockHashLookup != null)
{
m_blockHashLookup.Add(hash, size, volumeID);
}
}
else if (currentVolumeId == -1)
{
//Update
m_updateBlockVolumeCommand.Transaction = transaction;
m_updateBlockVolumeCommand.SetParameterValue(0, volumeID);
m_updateBlockVolumeCommand.SetParameterValue(1, hash);
m_updateBlockVolumeCommand.SetParameterValue(2, size);
var c = m_updateBlockVolumeCommand.ExecuteNonQuery();
if (c != 1)
{
throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c, hash, size));
}
if (m_blockHashLookup != null)
{
m_blockHashLookup.Add(hash, size, volumeID);
}
}
else
{
m_insertDuplicateBlockCommand.Transaction = transaction;
m_insertDuplicateBlockCommand.SetParameterValue(0, hash);
m_insertDuplicateBlockCommand.SetParameterValue(1, size);
m_insertDuplicateBlockCommand.SetParameterValue(2, volumeID);
m_insertDuplicateBlockCommand.ExecuteNonQuery();
}
}
/// <summary>
/// Calculates the sequence in which files should be deleted based on their releations.
/// </summary>
/// <returns>The deletable volumes.</returns>
/// <param name="deleteableVolumes">Block volumes slated for deletion.</param>
public IEnumerable <IRemoteVolume> GetDeletableVolumes(IEnumerable <IRemoteVolume> deleteableVolumes, System.Data.IDbTransaction transaction)
{
using (var cmd = m_connection.CreateCommand())
{
// Although the generated index volumes are always in pairs,
// this code handles many-to-many relations between
// index files and block volumes, should this be added later
var lookupBlock = new Dictionary <string, List <IRemoteVolume> >();
var lookupIndexfiles = new Dictionary <string, List <string> >();
cmd.Transaction = transaction;
using (var rd = cmd.ExecuteReader(@"SELECT ""C"".""Name"", ""B"".""Name"", ""B"".""Hash"", ""B"".""Size"" FROM ""IndexBlockLink"" A, ""RemoteVolume"" B, ""RemoteVolume"" C WHERE ""A"".""IndexVolumeID"" = ""B"".""ID"" AND ""A"".""BlockVolumeID"" = ""C"".""ID"" AND ""B"".""Hash"" IS NOT NULL AND ""B"".""Size"" IS NOT NULL "))
while (rd.Read())
{
var name = rd.GetValue(0).ToString();
List <IRemoteVolume> indexfileList;
if (!lookupBlock.TryGetValue(name, out indexfileList))
{
indexfileList = new List <IRemoteVolume>();
lookupBlock.Add(name, indexfileList);
}
var v = new RemoteVolume(rd.GetString(1), rd.GetString(2), rd.GetInt64(3));
indexfileList.Add(v);
List <string> blockList;
if (!lookupIndexfiles.TryGetValue(v.Name, out blockList))
{
blockList = new List <string>();
lookupIndexfiles.Add(v.Name, blockList);
}
blockList.Add(name);
}
foreach (var r in deleteableVolumes.Distinct())
{
// Return the input
yield return(r);
List <IRemoteVolume> indexfileList;
if (lookupBlock.TryGetValue(r.Name, out indexfileList))
{
foreach (var sh in indexfileList)
{
List <string> backref;
if (lookupIndexfiles.TryGetValue(sh.Name, out backref))
{
//If this is the last reference,
// remove the index file as well
if (backref.Remove(r.Name) && backref.Count == 0)
{
yield return(sh);
}
}
}
}
}
}
}
/// <summary>
/// Construtor padrão.
/// </summary>
/// <param name="connection"></param>
/// <param name="transaction"></param>
public DbTransactionWrapper(System.Data.IDbConnection connection, System.Data.IDbTransaction transaction)
{
_connection = connection;
_transaction = transaction;
}
/// <summary>
/// Drops all entries related to operations listed in the table.
/// </summary>
/// <param name="toDelete">The fileset entries to delete</param>
/// <param name="transaction">The transaction to execute the commands in</param>
/// <returns>A list of filesets to delete</returns>
public IEnumerable <KeyValuePair <string, long> > DropFilesetsFromTable(DateTime[] toDelete, System.Data.IDbTransaction transaction)
{
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = transaction;
var q = "";
foreach (var n in toDelete)
{
if (q.Length == 0)
{
q = "?";
}
else
{
q += ",?";
}
}
//First we remove unwanted entries
cmd.ExecuteNonQuery(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" IN (" + q + @")) ", toDelete.Select(x => NormalizeDateTimeToEpochSeconds(x)).Cast <object>().ToArray());
var deleted = cmd.ExecuteNonQuery(@"DELETE FROM ""Fileset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FilesetID"" FROM ""FilesetEntry"") ");
if (deleted != toDelete.Length)
{
throw new Exception(string.Format("Unexpected number of deleted filesets {0} vs {1}", deleted, toDelete.Length));
}
//Then we delete anything that is no longer being referenced
cmd.ExecuteNonQuery(@"DELETE FROM ""File"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""File"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""File"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") ");
//We save the block info for the remote files, before we delete it
cmd.ExecuteNonQuery(@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""Block"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlockID"" AS ""BlockID"" FROM ""BlocksetEntry"" UNION SELECT DISTINCT ""ID"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Block"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlockID"" FROM ""BlocksetEntry"" UNION SELECT DISTINCT ""ID"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"") ");
//Find all remote filesets that are no longer required, and mark them as delete
var updated = cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""State"" = ? WHERE ""Type"" = ? AND ""State"" IN (?, ?) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"") ", RemoteVolumeState.Deleting.ToString(), RemoteVolumeType.Files.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
if (deleted != updated)
{
throw new Exception(string.Format("Unexpected number of remote volumes marked as deleted. Found {0} filesets, but {1} volumes", deleted, updated));
}
using (var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" = ? ", RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleting.ToString()))
while (rd.Read())
{
yield return(new KeyValuePair <string, long>(rd.GetString(0), rd.ConvertValueToInt64(1)));
}
}
}
public ICompactReport GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, System.Data.IDbTransaction transaction)
{
return(new CompactReport(volsize, wastethreshold, smallfilesize, maxsmallfilecount, GetWastedSpaceReport(transaction).ToList()));
}
private string Add(RequestTPLAdd request, System.Data.IDbTransaction ts, MM_UserInfo user)
{
AccountSetBLL ac = new AccountSetBLL(cnn);
var period = ac.GetActivePeriod(user.AccountId, ts);
decimal total = 0;
var flag = 0;
//凭证上辅助核算项直接写死对应列
foreach (var item in request.Detail)
{
if (string.IsNullOrEmpty(item.SubjectCode))
{
continue;
}
item.Id = Guid.NewGuid().ToString("N");
#region 辅助核算赋值
string[] cals = item.CalValue1.Split(new char[] { '#' });
for (int i = 0; i < cals.Length; i++)
{
if (string.IsNullOrEmpty(cals[i]))
{
continue;
}
string[] ss = cals[i].Split(new char[] { ',' });
if (i == 0)
{
item.CalItem1 = ss[0];
item.CalValue1 = ss[1];
}
else if (i == 1)
{
item.CalItem2 = ss[0];
item.CalValue2 = ss[1];
}
else if (i == 2)
{
item.CalItem3 = ss[0];
item.CalValue3 = ss[1];
}
else if (i == 3)
{
item.CalItem4 = ss[0];
item.CalValue4 = ss[1];
}
else if (i == 4)
{
item.CalItem5 = ss[0];
item.CalValue5 = ss[1];
}
}
#endregion
total += item.Money_Debit;
item.Year = period.Year;
item.ParentId = request.Head.Id;
item.AccountId = user.AccountId;
cnn.Insert <TKS_FAS_DocDetailTPL>(item, ts);
flag++;
}
if (flag == 0)
{
throw new NormalException("亲，您还没有填写明细");
}
request.Head.AccountId = user.AccountId;
request.Head.AMT_DBT = total;
request.Head.IsCarry = 0;
request.Head.TPLTarget = 0; //用户选择
request.Head.IsCustom = 1; //自定义
request.Head.CreateUser = user.User.TrueName;
request.Head.CreateDate = DateTime.Now;
cnn.Insert <TKS_FAS_DocTPL>(request.Head, ts);
return(request.Head.Id);
}
public void Dispose()
{
if (m_insertPreviousElementCommand != null)
{
try { m_insertPreviousElementCommand.Dispose(); }
catch {}
finally { m_insertPreviousElementCommand = null; }
}
if (m_insertCurrentElementCommand != null)
{
try { m_insertCurrentElementCommand.Dispose(); }
catch {}
finally { m_insertCurrentElementCommand = null; }
}
if (m_transaction != null)
{
try { m_transaction.Rollback(); }
catch {}
finally
{
m_previousTable = null;
m_currentTable= null;
m_transaction = null;
}
}
}
public void FindMissingBlocklistHashes(long hashsize, long blocksize, System.Data.IDbTransaction transaction)
{
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = transaction;
//Update all small blocklists and matching blocks
var selectSmallBlocks = string.Format(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist);
var selectBlockHashes = string.Format(
@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " +
SELECT_BLOCKLIST_ENTRIES +
@" )",
blocksize,
hashsize,
m_tempblocklist,
blocksize / hashsize
);
var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )";
var selectNewBlocks = string.Format(
@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" " +
@" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" +
@" LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" )" +
@" WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" ",
selectAllBlocks
);
var insertBlocksCommand =
@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " +
selectNewBlocks;
// Insert all known blocks into block table with volumeid = -1
cmd.ExecuteNonQuery(insertBlocksCommand);
var selectBlocklistBlocksetEntries = string.Format(
@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " +
SELECT_BLOCKLIST_ENTRIES +
@") D, ""BlocklistHash"" E, ""Block"" F, ""Block"" G WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""BlocklistSize"" = ""G"".""Size"" AND ""D"".""BlocklistHash"" = ""G"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" ",
// TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati
// this could be used instead
//@") D, ""Block"" WHERE ""BlockQuery"".""BlockHash"" = ""Block"".""Hash"" AND ""BlockQuery"".""BlockSize"" = ""Block"".""Size"" ";
blocksize,
hashsize,
m_tempblocklist,
blocksize / hashsize
);
var selectBlocksetEntries = string.Format(
@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{1}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {0} ",
blocksize,
m_tempsmalllist
);
var selectAllBlocksetEntries =
selectBlocklistBlocksetEntries +
@" UNION " +
selectBlocksetEntries;
var selectFiltered =
@"SELECT DISTINCT ""H"".""BlocksetID"", ""H"".""Index"", ""H"".""BlockID"" FROM (" +
selectAllBlocksetEntries +
@") H WHERE (""H"".""BlocksetID"" || ':' || ""H"".""Index"") NOT IN (SELECT (""ExistingBlocksetEntries"".""BlocksetID"" || ':' || ""ExistingBlocksetEntries"".""Index"") FROM ""BlocksetEntry"" ""ExistingBlocksetEntries"" )";
var insertBlocksetEntriesCommand =
@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered;
try
{
cmd.ExecuteNonQuery(insertBlocksetEntriesCommand);
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "BlocksetInsertFailed", ex, "Blockset insert failed, comitting temporary tables for trace purposes");
using (var fixcmd = m_connection.CreateCommand())
{
fixcmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""{0}-Failure"" AS SELECT * FROM ""{0}"" ", m_tempblocklist));
fixcmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""{0}-Failure"" AS SELECT * FROM ""{0}"" ", m_tempsmalllist));
}
throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis");
}
}
}
private bool Deleteitemone(iSqlTransaction trans, int id)
{
try
{
if (trans == null) return false;
string SQL = "DELETE FROM " + TABLENAME + " WHERE id=@ID";
iSqlParameter[] parms = new iSqlParameter[]{
new iSqlParameter(PARM_ID, iSqlType.Field_tInterger),
};
int i = -1;
parms[++i].Value = id;
HELPER.executeNonQuery(trans, iCommandType.Text, SQL, parms);
return true;
}
catch (Exception ex)
{
throw ex;
}
}
public void AddDirectoryEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
{
AddEntry(filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction);
}
private bool Deleterecursive(iSqlTransaction trans, string iid)
{
try
{
if (trans == null || string.IsNullOrEmpty(iid)) return false;
string SQL_SELECT = "SELECT id FROM " + TABLENAME + " WHERE pid IN (" + iid + ")";
string strIid = "";
using (iSqlDataReader dar = HELPER.executeReader(trans, SQL_SELECT))
{
while (dar.Read())
{
int id = dar.IsDBNull(0) ? -1 : dar.GetInt32(0);
strIid += id + ",";
}
if (!string.IsNullOrEmpty(strIid))
strIid = strIid.Remove(strIid.LastIndexOf(","), 1);
}
this.Deleterecursive(trans, strIid);
string SQL_DELETE = "UPDATE " + TABLENAME + " SET status=" + (int)CConstants.State.Status.Deleted + " WHERE id IN (" + iid + ")";
HELPER.executeNonQuery(trans, SQL_DELETE);
SQL_DELETE = "UPDATE " + TABLENAME + " SET status=" + (int)CConstants.State.Status.Deleted + " WHERE pid IN (" + iid + ")";
HELPER.executeNonQuery(trans, SQL_DELETE);
return true;
}
catch (Exception ex)
{
throw ex;
}
}
public void AddSymlinkEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
{
AddEntry(filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction);
}
private bool Saveitem(iSqlTransaction trans, ItemcategoryattrInfo info)
{
try
{
if (trans == null || info == null) return false;
string SQL = SQL_INSERT;
iSqlParameter[] parms = this.getParameter(SQL);
this.setParameter(parms, info);
HELPER.executeNonQuery(trans, iCommandType.Text, SQL, parms);
return true;
}
catch (Exception ex)
{
throw ex;
}
}
public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
{
AddEntry(filesetid, path, time, blocksetid, metadataid, transaction);
}
private bool Saveitem(iSqlTransaction trans, MeCreditcardHistoryInfo info)
{
try
{
if (trans == null || info == null) return false;
string SQL = string.Empty;
if (info.Id == 0)
{
SQL = SQL_INSERT;
info.Id = (int)HELPER.getNewID(trans, TABLENAME);
iSqlParameter[] parms = this.getParameter(SQL);
this.setParameter(parms, info);
HELPER.executeNonQuery(trans, iCommandType.Text, SQL, parms);
}
else if (HELPER.isExist(trans, TABLENAME, info.Id))
{
SQL = SQL_UPDATE;
iSqlParameter[] parms = this.getParameter(SQL);
this.setParameter(parms, info);
HELPER.executeNonQuery(trans, iCommandType.Text, SQL, parms);
}
else
{
SQL = SQL_INSERT;
info.Id = (int)HELPER.getNewID(trans, TABLENAME);
iSqlParameter[] parms = this.getParameter(SQL);
this.setParameter(parms, info);
HELPER.executeNonQuery(trans, iCommandType.Text, SQL, parms);
}
return true;
}
catch (Exception ex)
{
throw ex;
}
}
public Transaction(System.Data.IDbTransaction transaction)
{
this.transaction = transaction;
}