public override void Run()
{
InvokeInfo("begin restore portal");
List<IModuleSpecifics> modulesToProcess = GetModulesToProcess().ToList();
InitProgress(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count);
using (var dataReader = new ZipReadOperator(BackupFilePath))
{
InvokeInfo("begin restore data");
var dbFactory = new DbFactory(ConfigPath);
foreach (var module in modulesToProcess)
{
var restoreTask = new RestoreDbModuleTask(module, dataReader, _columnMapper, dbFactory, ReplaceDate);
foreach (var tableName in IgnoredTables)
{
restoreTask.IgnoreTable(tableName);
}
RunSubtask(restoreTask);
}
InvokeInfo("end restore data");
if (ProcessStorage)
DoRestoreStorage(dataReader);
if (UnblockPortalAfterCompleted)
SetTenantActive(dbFactory, _columnMapper.GetTenantMapping());
}
InvokeInfo("end restore portal");
}
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory)
{
if (reader == null)
throw new ArgumentNullException("reader");
if (columnMapper == null)
throw new ArgumentNullException("columnMapper");
if (factory == null)
throw new ArgumentNullException("factory");
_module = module;
_reader = reader;
_columnMapper = columnMapper;
_factory = factory;
}
public override void Run()
{
InvokeInfo("begin delete portal ({0})", Tenant.TenantAlias);
List<IModuleSpecifics> modulesToProcess = GetModulesToProcess().Reverse().ToList();
InitProgress(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count);
var dbFactory = new DbFactory(ConfigPath);
foreach (var module in modulesToProcess)
{
DoDeleteModule(dbFactory, module);
}
if (ProcessStorage)
{
DoDeleteStorage();
}
InvokeInfo("end delete portal ({0})", Tenant.TenantAlias);
}
private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module)
{
InvokeInfo("begin saving data for module {0}", module.ModuleName);
int tablesCount = module.Tables.Count();
int tablesProcessed = 0;
using (var connection = dbFactory.OpenConnection(module.ConnectionStringName))
{
foreach (var table in module.Tables)
{
InvokeInfo("begin saving table {0}", table.Name);
using (var data = new DataTable(table.Name))
{
ActionInvoker.Try(
state =>
{
data.Clear();
var t = (TableInfo)state;
var dataAdapter = dbFactory.CreateDataAdapter(module.ConnectionStringName);
dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(600);
((DbDataAdapter)dataAdapter).Fill(data);
},
table,
maxAttempts: 5,
onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); },
onAttemptFailure: error => InvokeWarning("backup attempt failure: {0}", error));
foreach (var col in data.Columns.Cast<DataColumn>().Where(col => col.DataType == typeof(DateTime)))
{
col.DateTimeMode = DataSetDateTime.Unspecified;
}
var stream = writer.BeginWriteEntry(KeyHelper.GetTableZipKey(module, data.TableName));
data.WriteXml(stream, XmlWriteMode.WriteSchema);
writer.EndWriteEntry();
data.Clear();
}
SetStepProgress((int)((++tablesProcessed*100)/(double)tablesCount));
}
}
InvokeInfo("end saving data for module {0}", module.ModuleName);
}
private void DoDeleteModule(DbFactory dbFactory, IModuleSpecifics module)
{
InvokeInfo("begin delete data for module ({0})", module.ModuleName);
int tablesCount = module.Tables.Count();
int tablesProcessed = 0;
using (var connection = dbFactory.OpenConnection(module.ConnectionStringName))
{
foreach (var table in module.GetTablesOrdered().Reverse().Where(t => !IgnoredTables.Contains(t.Name)))
{
ActionInvoker.Try(state =>
{
var t = (TableInfo)state;
module.CreateDeleteCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(120).ExecuteNonQuery();
}, table, 5, onFailure: error => { throw ThrowHelper.CantDeleteTable(table.Name, error); });
SetStepProgress((int)((++tablesProcessed*100)/(double)tablesCount));
}
}
InvokeInfo("end delete data for module ({0})", module.ModuleName);
}
public override void Run()
{
InvokeInfo("begin backup portal ({0})", Tenant.TenantAlias);
List<IModuleSpecifics> modulesToProcess = GetModulesToProcess().ToList();
InitProgress(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count);
using (var writer = new ZipWriteOperator(BackupFilePath))
{
var dbFactory = new DbFactory(ConfigPath);
foreach (var module in modulesToProcess)
{
DoBackupModule(writer, dbFactory, module);
}
if (ProcessStorage)
{
DoBackupStorage(writer);
}
}
InvokeInfo("end backup portal ({0})", Tenant.TenantAlias);
}
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (columnMapper == null)
{
throw new ArgumentNullException("columnMapper");
}
if (factory == null)
{
throw new ArgumentNullException("factory");
}
_module = module;
_reader = reader;
_columnMapper = columnMapper;
_factory = factory;
}
public override void RunJob()
{
Logger.Debug("begin restore portal");
Logger.Debug("begin restore data");
using (var dataReader = new ZipReadOperator(BackupFilePath))
{
using (var entry = dataReader.GetEntry(KeyHelper.GetDumpKey()))
{
Dump = entry != null && CoreContext.Configuration.Standalone;
}
var dbFactory = new DbFactory(ConfigPath);
if (Dump)
{
RestoreFromDump(dataReader);
}
else
{
var modulesToProcess = GetModulesToProcess().ToList();
SetStepsCount(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count);
foreach (var module in modulesToProcess)
{
var restoreTask = new RestoreDbModuleTask(Logger, module, dataReader, _columnMapper, dbFactory, ReplaceDate, Dump);
restoreTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress);
foreach (var tableName in IgnoredTables)
{
restoreTask.IgnoreTable(tableName);
}
restoreTask.RunJob();
}
}
Logger.Debug("end restore data");
if (ProcessStorage)
{
if (CoreContext.Configuration.Standalone)
{
Logger.Debug("clear cache");
AscCache.ClearCache();
}
DoRestoreStorage(dataReader);
}
if (UnblockPortalAfterCompleted)
{
SetTenantActive(dbFactory, _columnMapper.GetTenantMapping());
}
}
if (CoreContext.Configuration.Standalone)
{
Logger.Debug("refresh license");
try
{
LicenseReader.RejectLicense();
}
catch (Exception ex)
{
Logger.Error(ex);
}
Logger.Debug("clear cache");
AscCache.ClearCache();
}
Logger.Debug("end restore portal");
}
public override void Run()
{
InvokeInfo("begin transfer portal ({0})", Tenant.TenantAlias);
string backupFilePath = GetBackupFilePath();
var fromDbFactory = new DbFactory(ConfigPath);
var toDbFactory = new DbFactory(ToConfigPath);
var columnMapper = new ColumnMapper();
try
{
//target db can have error tenant from the previous attempts
SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d"));
if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, TenantStatus.Transfering);
}
InitProgress(ProcessStorage ? 3 : 2);
//save db data to temporary file
var backupTask = new BackupPortalTask(Tenant, ConfigPath, backupFilePath)
{
ProcessStorage = false
};
foreach (var moduleName in IgnoredModules)
{
backupTask.IgnoreModule(moduleName);
}
RunSubtask(backupTask);
//restore db data from temporary file
var restoreTask = new RestorePortalTask(ToConfigPath, backupFilePath, columnMapper)
{
ProcessStorage = false
};
foreach (var moduleName in IgnoredModules)
{
restoreTask.IgnoreModule(moduleName);
}
RunSubtask(restoreTask);
//transfer files
if (ProcessStorage)
{
DoTransferStorage(columnMapper);
}
SaveTenant(toDbFactory, TenantStatus.Active);
if (DeleteOldPortalAfterCompletion)
{
SaveTenant(fromDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_deleted");
}
else if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, TenantStatus.Active);
}
}
catch
{
SaveTenant(fromDbFactory, TenantStatus.Active);
if (columnMapper.GetTenantMapping() > 0)
{
SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error");
}
throw;
}
finally
{
if (DeleteBackupFileAfterCompletion)
{
File.Delete(backupFilePath);
}
InvokeInfo("end transfer portal ({0})", Tenant.TenantAlias);
}
}
private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module)
{
Logger.DebugFormat("begin saving data for module {0}", module.ModuleName);
var tablesToProcess = module.Tables.Where(t => !IgnoredTables.Contains(t.Name) && t.InsertMethod != InsertMethod.None).ToList();
var tablesCount = tablesToProcess.Count;
var tablesProcessed = 0;
using (var connection = dbFactory.OpenConnection())
{
foreach (var table in tablesToProcess)
{
Logger.DebugFormat("begin load table {0}", table.Name);
using (var data = new DataTable(table.Name))
{
ActionInvoker.Try(
state =>
{
data.Clear();
int counts;
var offset = 0;
do
{
var t = (TableInfo)state;
var dataAdapter = dbFactory.CreateDataAdapter();
dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), TenantId, t, Limit, offset).WithTimeout(600);
counts = ((DbDataAdapter)dataAdapter).Fill(data);
offset += Limit;
} while (counts == Limit);
},
table,
maxAttempts: 5,
onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); },
onAttemptFailure: error => Logger.Warn("backup attempt failure: {0}", error));
foreach (var col in data.Columns.Cast <DataColumn>().Where(col => col.DataType == typeof(DateTime)))
{
col.DateTimeMode = DataSetDateTime.Unspecified;
}
module.PrepareData(data);
Logger.DebugFormat("end load table {0}", table.Name);
Logger.DebugFormat("begin saving table {0}", table.Name);
var tmp = Path.GetTempFileName();
using (var file = File.OpenWrite(tmp))
{
data.WriteXml(file, XmlWriteMode.WriteSchema);
data.Clear();
}
writer.WriteEntry(KeyHelper.GetTableZipKey(module, data.TableName), tmp);
File.Delete(tmp);
Logger.DebugFormat("end saving table {0}", table.Name);
}
SetCurrentStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount));
}
}
Logger.DebugFormat("end saving data for module {0}", module.ModuleName);
}
public override void RunJob()
{
Logger.DebugFormat("begin transfer {0}", TenantId);
var fromDbFactory = new DbFactory(null, null);
var toDbFactory = new DbFactory(null, null);
var tenantAlias = GetTenantAlias(fromDbFactory);
var backupFilePath = GetBackupFilePath(tenantAlias);
var columnMapper = new ColumnMapper();
try
{
//target db can have error tenant from the previous attempts
SaveTenant(toDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d"));
if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Transfering);
}
SetStepsCount(ProcessStorage ? 3 : 2);
//save db data to temporary file
var backupTask = ServiceProvider.GetService <BackupPortalTask>();
backupTask.Init(TenantId, ConfigPath, backupFilePath, Limit);
backupTask.ProcessStorage = false;
backupTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress);
foreach (var moduleName in IgnoredModules)
{
backupTask.IgnoreModule(moduleName);
}
backupTask.RunJob();
//restore db data from temporary file
var restoreTask = ServiceProvider.GetService <RestorePortalTask>();
restoreTask.Init(ToConfigPath, backupFilePath, columnMapper: columnMapper);
restoreTask.ProcessStorage = false;
restoreTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress);
foreach (var moduleName in IgnoredModules)
{
restoreTask.IgnoreModule(moduleName);
}
restoreTask.RunJob();
//transfer files
if (ProcessStorage)
{
DoTransferStorage(columnMapper);
}
SaveTenant(toDbFactory, tenantAlias, TenantStatus.Active);
if (DeleteOldPortalAfterCompletion)
{
SaveTenant(fromDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_deleted");
}
else if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Active);
}
}
catch
{
SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Active);
if (columnMapper.GetTenantMapping() > 0)
{
SaveTenant(toDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_error");
}
throw;
}
finally
{
if (DeleteBackupFileAfterCompletion)
{
File.Delete(backupFilePath);
}
Logger.DebugFormat("end transfer {0}", TenantId);
}
}
public DeletePortalTask(DbFactory dbFactory, IOptionsMonitor <ILog> options, int tenantId, string configPath, StorageFactory storageFactory, StorageFactoryConfig storageFactoryConfig, ModuleProvider moduleProvider)
: base(dbFactory, options, storageFactory, storageFactoryConfig, moduleProvider)
{
Init(tenantId, configPath);
}
public RestoreDbModuleTask(ILog logger, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump)
: base(logger, -1, null)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (columnMapper == null)
{
throw new ArgumentNullException("columnMapper");
}
if (factory == null)
{
throw new ArgumentNullException("factory");
}
_module = module;
_reader = reader;
_columnMapper = columnMapper;
_factory = factory;
_replaceDate = replaceDate;
this.dump = dump;
}
private static void SetTenantActive(DbFactory dbFactory, int tenantId)
{
using (var connection = dbFactory.OpenConnection("core"))
{
var commandText = string.Format(
"update tenants_tenants " +
"set " +
" status={0}, " +
" last_modified='{1}', " +
" statuschanged='{1}' " +
"where id = '{2}'",
(int)TenantStatus.Active,
DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss"),
tenantId);
connection.CreateCommand(commandText).WithTimeout(120).ExecuteNonQuery();
}
}
private void DumpTableData(string t, string dir, int count)
{
try
{
if (count == 0)
{
Logger.DebugFormat("dump table data stop {0}", t);
SetStepCompleted(2);
return;
}
Logger.DebugFormat("dump table data start {0}", t);
var searchWithPrimary = false;
string primaryIndex;
var primaryIndexStep = 0;
var primaryIndexStart = 0;
List <string> columns;
using (var connection = DbFactory.OpenConnection())
{
var command = connection.CreateCommand();
command.CommandText = string.Format("SHOW COLUMNS FROM `{0}`", t);
columns = ExecuteList(command).Select(r => "`" + Convert.ToString(r[0]) + "`").ToList();
if (command.CommandText.Contains("tenants_quota") || command.CommandText.Contains("webstudio_settings"))
{
}
}
using (var connection = DbFactory.OpenConnection())
{
var command = connection.CreateCommand();
command.CommandText = string.Format("select COLUMN_NAME from information_schema.`COLUMNS` where TABLE_SCHEMA = '{0}' and TABLE_NAME = '{1}' and COLUMN_KEY = 'PRI' and DATA_TYPE = 'int'", connection.Database, t);
primaryIndex = ExecuteList(command).ConvertAll(r => Convert.ToString(r[0])).FirstOrDefault();
}
using (var connection = DbFactory.OpenConnection())
{
var command = connection.CreateCommand();
command.CommandText = string.Format("SHOW INDEXES FROM {0} WHERE COLUMN_NAME='{1}' AND seq_in_index=1", t, primaryIndex);
var isLeft = ExecuteList(command);
searchWithPrimary = isLeft.Count == 1;
}
if (searchWithPrimary)
{
using var connection = DbFactory.OpenConnection();
var command = connection.CreateCommand();
command.CommandText = string.Format("select max({1}), min({1}) from {0}", t, primaryIndex);
var minMax = ExecuteList(command).ConvertAll(r => new Tuple <int, int>(Convert.ToInt32(r[0]), Convert.ToInt32(r[1]))).FirstOrDefault();
primaryIndexStart = minMax.Item2;
primaryIndexStep = (minMax.Item1 - minMax.Item2) / count;
if (primaryIndexStep < Limit)
{
primaryIndexStep = Limit;
}
}
var path = Path.Combine(dir, t);
var offset = 0;
do
{
List <object[]> result;
if (searchWithPrimary)
{
result = GetDataWithPrimary(t, columns, primaryIndex, primaryIndexStart, primaryIndexStep);
primaryIndexStart += primaryIndexStep;
}
else
{
result = GetData(t, columns, offset);
}
offset += Limit;
var resultCount = result.Count;
if (resultCount == 0)
{
break;
}
SaveToFile(path, t, columns, result);
if (resultCount < Limit)
{
break;
}
} while (true);
SetStepCompleted();
Logger.DebugFormat("dump table data stop {0}", t);
}
catch (Exception e)
{
Logger.Error(e);
throw;
}
}
private void DoDump(IDataWriteOperator writer)
{
var tmp = Path.GetTempFileName();
File.AppendAllText(tmp, true.ToString());
writer.WriteEntry(KeyHelper.GetDumpKey(), tmp);
List <string> tables;
var files = new List <BackupFileInfo>();
using (var connection = DbFactory.OpenConnection())
{
var command = connection.CreateCommand();
command.CommandText = "show tables";
tables = ExecuteList(command).Select(r => Convert.ToString(r[0])).ToList();
}
/* using (var dbManager = new DbManager("default", 100000))
* {
* tables = dbManager.ExecuteList("show tables;").Select(r => Convert.ToString(r[0])).ToList();
* }*/
var stepscount = tables.Count * 4; // (schema + data) * (dump + zip)
if (ProcessStorage)
{
var tenants = TenantManager.GetTenants(false).Select(r => r.TenantId);
foreach (var t in tenants)
{
files.AddRange(GetFiles(t));
}
stepscount += files.Count * 2 + 1;
Logger.Debug("files:" + files.Count);
}
SetStepsCount(stepscount);
var excluded = ModuleProvider.AllModules.Where(r => IgnoredModules.Contains(r.ModuleName)).SelectMany(r => r.Tables).Select(r => r.Name).ToList();
excluded.AddRange(IgnoredTables);
excluded.Add("res_");
var dir = Path.GetDirectoryName(BackupFilePath);
var subDir = Path.Combine(dir, Path.GetFileNameWithoutExtension(BackupFilePath));
var schemeDir = Path.Combine(subDir, KeyHelper.GetDatabaseSchema());
var dataDir = Path.Combine(subDir, KeyHelper.GetDatabaseData());
if (!Directory.Exists(schemeDir))
{
Directory.CreateDirectory(schemeDir);
}
if (!Directory.Exists(dataDir))
{
Directory.CreateDirectory(dataDir);
}
var dict = tables.ToDictionary(t => t, SelectCount);
tables.Sort((pair1, pair2) => dict[pair1].CompareTo(dict[pair2]));
for (var i = 0; i < tables.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < tables.Count; j++)
{
var t = tables[i + j];
tasks.Add(Task.Run(() => DumpTableScheme(t, schemeDir)));
if (!excluded.Any(t.StartsWith))
{
tasks.Add(Task.Run(() => DumpTableData(t, dataDir, dict[t])));
}
else
{
SetStepCompleted(2);
}
}
Task.WaitAll(tasks.ToArray());
ArchiveDir(writer, subDir);
}
Logger.DebugFormat("dir remove start {0}", subDir);
Directory.Delete(subDir, true);
Logger.DebugFormat("dir remove end {0}", subDir);
if (ProcessStorage)
{
DoDumpStorage(writer, files);
}
}
public override void Run()
{
InvokeInfo("begin transfer portal ({0})", Tenant.TenantAlias);
string backupFilePath = GetBackupFilePath();
var fromDbFactory = new DbFactory(ConfigPath);
var toDbFactory = new DbFactory(ToConfigPath);
var columnMapper = new ColumnMapper();
try
{
//target db can have error tenant from the previous attempts
SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d"));
if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, TenantStatus.Transfering);
}
InitProgress(ProcessStorage ? 3 : 2);
//save db data to temporary file
var backupTask = new BackupPortalTask(Tenant, ConfigPath, backupFilePath) {ProcessStorage = false};
foreach (var moduleName in IgnoredModules)
{
backupTask.IgnoreModule(moduleName);
}
RunSubtask(backupTask);
//restore db data from temporary file
var restoreTask = new RestorePortalTask(ToConfigPath, backupFilePath, columnMapper) {ProcessStorage = false};
foreach (var moduleName in IgnoredModules)
{
restoreTask.IgnoreModule(moduleName);
}
RunSubtask(restoreTask);
//transfer files
if (ProcessStorage)
{
DoTransferStorage(columnMapper);
}
SaveTenant(toDbFactory, TenantStatus.Active);
if (DeleteOldPortalAfterCompletion)
{
SaveTenant(fromDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_deleted");
}
else if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, TenantStatus.Active);
}
}
catch
{
SaveTenant(fromDbFactory, TenantStatus.Active);
if (columnMapper.GetTenantMapping() > 0)
{
SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error");
}
throw;
}
finally
{
if (DeleteBackupFileAfterCompletion)
{
File.Delete(backupFilePath);
}
InvokeInfo("end transfer portal ({0})", Tenant.TenantAlias);
}
}
private void SaveTenant(DbFactory dbFactory, TenantStatus status, string newAlias = null, string whereCondition = null)
{
using (var connection = dbFactory.OpenConnection("core"))
{
if (newAlias == null)
{
newAlias = Tenant.TenantAlias;
}
else if (newAlias != Tenant.TenantAlias)
{
newAlias = GetUniqAlias(connection, newAlias);
}
var commandText = string.Format(
"update tenants_tenants " +
"set " +
" status={0}, " +
" alias = '{1}', " +
" last_modified='{2}', " +
" statuschanged='{2}' " +
"where alias = '{3}'",
status.ToString("d"),
newAlias,
DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss"),
Tenant.TenantAlias);
if (!string.IsNullOrEmpty(whereCondition))
commandText += (" and " + whereCondition);
connection.CreateCommand(commandText).WithTimeout(120).ExecuteNonQuery();
}
}
