private void DoBackupStorage(IDataWriteOperator writer, List <IGrouping <string, BackupFileInfo> > fileGroups)
{
Logger.Debug("begin backup storage");
foreach (var group in fileGroups)
{
var filesProcessed = 0;
var filesCount = group.Count();
var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), group.Key);
foreach (var file in group)
{
ActionInvoker.Try(state =>
{
var f = (BackupFileInfo)state;
using (var fileStream = storage.GetReadStream(f.Domain, f.Path))
{
var tmp = Path.GetTempFileName();
try
{
using (var tmpFile = File.OpenWrite(tmp))
{
fileStream.CopyTo(tmpFile);
}
writer.WriteEntry(KeyHelper.GetFileZipKey(file), tmp);
}
finally
{
if (File.Exists(tmp))
{
File.Delete(tmp);
}
}
}
}, file, 5, error => Logger.Warn("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));
SetCurrentStepProgress((int)(++filesProcessed * 100 / (double)filesCount));
}
}
var restoreInfoXml = new XElement(
"storage_restore",
fileGroups
.SelectMany(group => group.Select(file => (object)file.ToXElement()))
.ToArray());
var tmpPath = Path.GetTempFileName();
using (var tmpFile = File.OpenWrite(tmpPath))
{
restoreInfoXml.WriteTo(tmpFile);
}
writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath);
File.Delete(tmpPath);
Logger.Debug("end backup storage");
}
public override void RunJob()
{
Logger.DebugFormat("begin restore data for module {0}", _module.ModuleName);
SetStepsCount(_module.Tables.Count(t => !IgnoredTables.Contains(t.Name)));
using (var connection = _factory.OpenConnection())
{
foreach (var table in _module.GetTablesOrdered().Where(t => !IgnoredTables.Contains(t.Name) && t.InsertMethod != InsertMethod.None))
{
Logger.DebugFormat("begin restore table {0}", table.Name);
var transactionsCommited = 0;
var rowsInserted = 0;
ActionInvoker.Try(
state =>
RestoreTable(connection.Fix(), (TableInfo)state, ref transactionsCommited,
ref rowsInserted), table, 5,
onAttemptFailure: error => _columnMapper.Rollback(),
onFailure: error => { throw ThrowHelper.CantRestoreTable(table.Name, error); });
SetStepCompleted();
Logger.DebugFormat("{0} rows inserted for table {1}", rowsInserted, table.Name);
}
}
Logger.DebugFormat("end restore data for module {0}", _module.ModuleName);
}
private void DoDeleteStorage(IEnumerable <string> storageModules, IEnumerable <Tenant> tenants)
{
Logger.Debug("begin delete storage");
foreach (var tenant in tenants)
{
foreach (var module in storageModules)
{
var storage = StorageFactory.GetStorage(ConfigPath, tenant.TenantId.ToString(), module);
var domains = StorageFactory.GetDomainList(ConfigPath, module).ToList();
domains.Add(string.Empty); //instead storage.DeleteFiles("\\", "*.*", true);
foreach (var domain in domains)
{
ActionInvoker.Try(
state =>
{
if (storage.IsDirectory((string)state))
{
storage.DeleteFiles((string)state, "\\", "*.*", true);
}
},
domain,
5,
onFailure: error => Logger.WarnFormat("Can't delete files for domain {0}: \r\n{1}", domain, error)
);
}
SetStepCompleted();
}
}
Logger.Debug("end delete storage");
}
private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module)
{
Logger.Debug("begin saving data for module {0}", module.ModuleName);
var tablesToProcess = module.Tables.Where(t => !IgnoredTables.Contains(t.Name) && t.InsertMethod != InsertMethod.None).ToList();
int tablesCount = tablesToProcess.Count;
int tablesProcessed = 0;
using (var connection = dbFactory.OpenConnection())
{
foreach (var table in tablesToProcess)
{
Logger.Debug("begin load table {0}", table.Name);
using (var data = new DataTable(table.Name))
{
ActionInvoker.Try(
state =>
{
data.Clear();
var t = (TableInfo)state;
var dataAdapter = dbFactory.CreateDataAdapter();
dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), TenantId, t).WithTimeout(600);
((DbDataAdapter)dataAdapter).Fill(data);
},
table,
maxAttempts: 5,
onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); },
onAttemptFailure: error => Logger.Warn("backup attempt failure: {0}", error));
foreach (var col in data.Columns.Cast <DataColumn>().Where(col => col.DataType == typeof(DateTime)))
{
col.DateTimeMode = DataSetDateTime.Unspecified;
}
module.PrepareData(data);
Logger.Debug("end load table {0}", table.Name);
Logger.Debug("begin saving table {0}", table.Name);
var tmp = Path.GetTempFileName();
using (var file = File.OpenWrite(tmp))
{
data.WriteXml(file, XmlWriteMode.WriteSchema);
data.Clear();
}
writer.WriteEntry(KeyHelper.GetTableZipKey(module, data.TableName), tmp);
File.Delete(tmp);
Logger.Debug("end saving table {0}", table.Name);
}
SetCurrentStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount));
}
}
Logger.Debug("end saving data for module {0}", module.ModuleName);
}
private void DoBackupStorage(IDataWriteOperator writer)
{
InvokeInfo("begin backup storage", Tenant.TenantAlias);
var fileGroups = GetFilesToProcess().GroupBy(file => file.Module).ToList();
int groupsProcessed = 0;
foreach (var group in fileGroups)
{
IDataStore storage = StorageFactory.GetStorage(ConfigPath, Tenant.TenantId.ToString(), group.Key, null, null);
foreach (BackupFileInfo file in group)
{
Stream stream = writer.BeginWriteEntry(KeyHelper.GetFileZipKey(file));
int offset = 0;
ActionInvoker.Try(state =>
{
var f = (BackupFileInfo)state;
using (var fileStream = storage.GetReadStream(f.Domain, f.Path, offset))
{
var buffer = new byte[2048];
int readed;
while ((readed = fileStream.Read(buffer, 0, buffer.Length)) > 0)
{
stream.Write(buffer, 0, readed);
offset += readed;
}
}
}, file, 5, error => InvokeWarning("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));
writer.EndWriteEntry();
}
SetStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
}
if (fileGroups.Count == 0)
{
SetStepCompleted();
}
var restoreInfoXml = new XElement(
"storage_restore",
fileGroups
.SelectMany(group => group.Select(file => (object)file.ToXElement()))
.ToArray());
Stream restoreInfoStream = writer.BeginWriteEntry(KeyHelper.GetStorageRestoreInfoZipKey());
restoreInfoXml.WriteTo(restoreInfoStream);
writer.EndWriteEntry();
InvokeInfo("end backup storage", Tenant.TenantAlias);
}
private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module)
{
InvokeInfo("begin saving data for module {0}", module.ModuleName);
int tablesCount = module.Tables.Count();
int tablesProcessed = 0;
using (var connection = dbFactory.OpenConnection(module.ConnectionStringName))
{
foreach (var table in module.Tables)
{
InvokeInfo("begin saving table {0}", table.Name);
using (var data = new DataTable(table.Name))
{
ActionInvoker.Try(
state =>
{
data.Clear();
var t = (TableInfo)state;
var dataAdapter = dbFactory.CreateDataAdapter(module.ConnectionStringName);
dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(600);
((DbDataAdapter)dataAdapter).Fill(data);
},
table,
maxAttempts: 5,
onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); },
onAttemptFailure: error => InvokeWarning("backup attempt failure: {0}", error));
foreach (var col in data.Columns.Cast <DataColumn>().Where(col => col.DataType == typeof(DateTime)))
{
col.DateTimeMode = DataSetDateTime.Unspecified;
}
var stream = writer.BeginWriteEntry(KeyHelper.GetTableZipKey(module, data.TableName));
data.WriteXml(stream, XmlWriteMode.WriteSchema);
writer.EndWriteEntry();
data.Clear();
}
SetStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount));
}
}
InvokeInfo("end saving data for module {0}", module.ModuleName);
}
private void DoDeleteStorage()
{
InvokeInfo("begin delete storage");
List <string> storageModules = StorageFactory.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed).ToList();
int modulesProcessed = 0;
foreach (string module in storageModules)
{
IDataStore storage = StorageFactory.GetStorage(ConfigPath, Tenant.TenantId.ToString(), module, null, null);
List <string> domains = StorageFactory.GetDomainList(ConfigPath, module).ToList();
foreach (var domain in domains)
{
ActionInvoker.Try(state => storage.DeleteFiles((string)state, "\\", "*.*", true), domain, 5,
onFailure: error => InvokeWarning("Can't delete files for domain {0}: \r\n{1}", domain, error));
}
storage.DeleteFiles("\\", "*.*", true);
SetStepProgress((int)((++modulesProcessed * 100) / (double)storageModules.Count));
}
InvokeInfo("end delete storage");
}
private void DoDeleteModule(DbFactory dbFactory, IModuleSpecifics module)
{
InvokeInfo("begin delete data for module ({0})", module.ModuleName);
int tablesCount = module.Tables.Count();
int tablesProcessed = 0;
using (var connection = dbFactory.OpenConnection(module.ConnectionStringName))
{
foreach (var table in module.GetTablesOrdered().Reverse().Where(t => !IgnoredTables.Contains(t.Name)))
{
ActionInvoker.Try(state =>
{
var t = (TableInfo)state;
module.CreateDeleteCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(120).ExecuteNonQuery();
}, table, 5, onFailure: error => { throw ThrowHelper.CantDeleteTable(table.Name, error); });
SetStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount));
}
}
InvokeInfo("end delete data for module ({0})", module.ModuleName);
}
private void DoDeleteStorage()
{
Logger.Debug("begin delete storage");
var storageModules = StorageFactoryConfig.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed).ToList();
var modulesProcessed = 0;
foreach (var module in storageModules)
{
var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), module);
var domains = StorageFactoryConfig.GetDomainList(ConfigPath, module).ToList();
foreach (var domain in domains)
{
ActionInvoker.Try(state => storage.DeleteFiles((string)state, "\\", "*.*", true), domain, 5,
onFailure: error => Logger.WarnFormat("Can't delete files for domain {0}: \r\n{1}", domain, error));
}
storage.DeleteFiles("\\", "*.*", true);
SetCurrentStepProgress((int)((++modulesProcessed * 100) / (double)storageModules.Count));
}
Logger.Debug("end delete storage");
}
private void DoBackupStorage(IDataWriteOperator writer, DbFactory dbFactory)
{
Logger.Debug("begin backup storage");
var files = GetFilesToProcess();
var exclude = new List <string>();
using (var db = dbFactory.OpenConnection())
using (var command = db.CreateCommand())
{
command.CommandText = "select storage_path from backup_backup where tenant_id = " + TenantId + " and storage_type = 0 and storage_path is not null";
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
exclude.Add(reader.GetString(0));
}
}
}
files = files.Where(f => !exclude.Any(e => f.Path.Contains(string.Format("/file_{0}/", e))));
var fileGroups = files.GroupBy(file => file.Module).ToList();
var groupsProcessed = 0;
foreach (var group in fileGroups)
{
var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), group.Key);
foreach (var file in group)
{
ActionInvoker.Try(state =>
{
var f = (BackupFileInfo)state;
using (var fileStream = storage.GetReadStream(f.Domain, f.Path))
{
var tmp = Path.GetTempFileName();
try
{
using (var tmpFile = File.OpenWrite(tmp))
{
fileStream.CopyTo(tmpFile);
}
writer.WriteEntry(KeyHelper.GetFileZipKey(file), tmp);
}
finally
{
if (File.Exists(tmp))
{
File.Delete(tmp);
}
}
}
}, file, 5, error => Logger.Warn("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));
}
SetCurrentStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
}
if (fileGroups.Count == 0)
{
SetStepCompleted();
}
var restoreInfoXml = new XElement(
"storage_restore",
fileGroups
.SelectMany(group => group.Select(file => (object)file.ToXElement()))
.ToArray());
var tmpPath = Path.GetTempFileName();
using (var tmpFile = File.OpenWrite(tmpPath))
{
restoreInfoXml.WriteTo(tmpFile);
}
writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath);
File.Delete(tmpPath);
Logger.Debug("end backup storage");
}
