public RestorePortalTask(string toConfigPath, string fromFilePath, ColumnMapper columnMapper = null)
: base(null, toConfigPath)
{
if (fromFilePath == null)
throw new ArgumentNullException("fromFilePath");
if (!File.Exists(fromFilePath))
throw new FileNotFoundException("file not found at given path");
BackupFilePath = fromFilePath;
_columnMapper = columnMapper ?? new ColumnMapper();
}
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory)
{
if (reader == null)
throw new ArgumentNullException("reader");
if (columnMapper == null)
throw new ArgumentNullException("columnMapper");
if (factory == null)
throw new ArgumentNullException("factory");
_module = module;
_reader = reader;
_columnMapper = columnMapper;
_factory = factory;
}
public static ColumnMapper ForRestoreDemoPortal(int tenantID)
{
var columnMapper = new ColumnMapper();
columnMapper.SetMapping("tenants_tenants", "id", tenantID);
//set up for projects:
columnMapper.SetMapping("projects_comments", "create_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_messages", "create_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_messages", "last_modified_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_milestones", "create_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_milestones", "last_modified_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_milestones", "deadline", DateTime.UtcNow.AddDays(7), val => !IsDefaultDateTime(val));
columnMapper.SetMapping("projects_projects", "create_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_projects", "last_modified_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_project_participant", "created", DateTime.UtcNow);
columnMapper.SetMapping("projects_project_participant", "updated", DateTime.UtcNow);
columnMapper.SetMapping("projects_report_template", "create_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_subtasks", "create_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_subtasks", "last_modified_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_tags", "create_on", DateTime.UtcNow, val => !IsDefaultDateTime(val));
columnMapper.SetMapping("projects_tags", "last_modified_on", DateTime.UtcNow, val => !IsDefaultDateTime(val));
columnMapper.SetMapping("projects_tasks", "deadline", DateTime.UtcNow.AddDays(7), val => !IsDefaultDateTime(val));
columnMapper.SetMapping("projects_tasks", "create_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_tasks", "last_modified_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_tasks", "start_date", DateTime.UtcNow);
columnMapper.SetMapping("projects_templates", "create_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_templates", "last_modified_on", DateTime.UtcNow);
columnMapper.SetMapping("projects_time_tracking", "create_on", DateTime.UtcNow);
//set up for crm:
columnMapper.SetMapping("crm_task", "deadline", DateTime.UtcNow.AddDays(7), val => !IsDefaultDateTime(val));
return columnMapper;
}
public override void RunJob()
{
Logger.Debug("begin transfer {0}", TenantId);
var fromDbFactory = new DbFactory(ConfigPath);
var toDbFactory = new DbFactory(ToConfigPath);
string tenantAlias = GetTenantAlias(fromDbFactory);
string backupFilePath = GetBackupFilePath(tenantAlias);
var columnMapper = new ColumnMapper();
try
{
//target db can have error tenant from the previous attempts
SaveTenant(toDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d"));
if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Transfering);
}
SetStepsCount(ProcessStorage ? 3 : 2);
//save db data to temporary file
var backupTask = new BackupPortalTask(Logger, TenantId, ConfigPath, backupFilePath, Limit)
{
ProcessStorage = false
};
backupTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress);
foreach (var moduleName in IgnoredModules)
{
backupTask.IgnoreModule(moduleName);
}
backupTask.RunJob();
//restore db data from temporary file
var restoreTask = new RestorePortalTask(Logger, ToConfigPath, backupFilePath, columnMapper)
{
ProcessStorage = false
};
restoreTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress);
foreach (var moduleName in IgnoredModules)
{
restoreTask.IgnoreModule(moduleName);
}
restoreTask.RunJob();
//transfer files
if (ProcessStorage)
{
DoTransferStorage(columnMapper);
}
SaveTenant(toDbFactory, tenantAlias, TenantStatus.Active);
if (DeleteOldPortalAfterCompletion)
{
SaveTenant(fromDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_deleted");
}
else if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Active);
}
}
catch
{
SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Active);
if (columnMapper.GetTenantMapping() > 0)
{
SaveTenant(toDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_error");
}
throw;
}
finally
{
if (DeleteBackupFileAfterCompletion)
{
File.Delete(backupFilePath);
}
Logger.Debug("end transfer {0}", TenantId);
}
}
public RestoreDbModuleTask(ILog logger, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump)
: base(logger, -1, null)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (columnMapper == null)
{
throw new ArgumentNullException("columnMapper");
}
if (factory == null)
{
throw new ArgumentNullException("factory");
}
_module = module;
_reader = reader;
_columnMapper = columnMapper;
_factory = factory;
_replaceDate = replaceDate;
this.dump = dump;
}
public override void Run()
{
InvokeInfo("begin transfer portal ({0})", Tenant.TenantAlias);
string backupFilePath = GetBackupFilePath();
var fromDbFactory = new DbFactory(ConfigPath);
var toDbFactory = new DbFactory(ToConfigPath);
var columnMapper = new ColumnMapper();
try
{
//target db can have error tenant from the previous attempts
SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d"));
if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, TenantStatus.Transfering);
}
InitProgress(ProcessStorage ? 3 : 2);
//save db data to temporary file
var backupTask = new BackupPortalTask(Tenant, ConfigPath, backupFilePath) {ProcessStorage = false};
foreach (var moduleName in IgnoredModules)
{
backupTask.IgnoreModule(moduleName);
}
RunSubtask(backupTask);
//restore db data from temporary file
var restoreTask = new RestorePortalTask(ToConfigPath, backupFilePath, columnMapper) {ProcessStorage = false};
foreach (var moduleName in IgnoredModules)
{
restoreTask.IgnoreModule(moduleName);
}
RunSubtask(restoreTask);
//transfer files
if (ProcessStorage)
{
DoTransferStorage(columnMapper);
}
SaveTenant(toDbFactory, TenantStatus.Active);
if (DeleteOldPortalAfterCompletion)
{
SaveTenant(fromDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_deleted");
}
else if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, TenantStatus.Active);
}
}
catch
{
SaveTenant(fromDbFactory, TenantStatus.Active);
if (columnMapper.GetTenantMapping() > 0)
{
SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error");
}
throw;
}
finally
{
if (DeleteBackupFileAfterCompletion)
{
File.Delete(backupFilePath);
}
InvokeInfo("end transfer portal ({0})", Tenant.TenantAlias);
}
}
private void DoTransferStorage(ColumnMapper columnMapper)
{
InvokeInfo("begin transfer storage");
var fileGroups = GetFilesToProcess().GroupBy(file => file.Module).ToList();
int groupsProcessed = 0;
foreach (var group in fileGroups)
{
ICrossModuleTransferUtility transferUtility =
StorageFactory.GetCrossModuleTransferUtility(
ConfigPath, Tenant.TenantId, group.Key,
ToConfigPath, columnMapper.GetTenantMapping(), group.Key);
foreach (BackupFileInfo file in group)
{
string adjustedPath = file.Path;
IModuleSpecifics module = ModuleProvider.GetByStorageModule(file.Module, file.Domain);
if (module == null || module.TryAdjustFilePath(columnMapper, ref adjustedPath))
{
try
{
transferUtility.CopyFile(file.Domain, file.Path, file.Domain, adjustedPath);
}
catch (Exception error)
{
InvokeWarning("Can't copy file ({0}:{1}): {2}", file.Module, file.Path, error);
}
}
else
{
InvokeWarning("Can't adjust file path \"{0}\".", file.Path);
}
}
SetStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
}
if (fileGroups.Count == 0)
SetStepCompleted();
InvokeInfo("end transfer storage");
}
private void DoRestoreStorage(IDataReadOperator dataReader)
{
Logger.Debug("begin restore storage");
var fileGroups = GetFilesToProcess(dataReader).GroupBy(file => file.Module).ToList();
var groupsProcessed = 0;
foreach (var group in fileGroups)
{
foreach (var file in group)
{
var storage = StorageFactory.GetStorage(ConfigPath, Dump ? file.Tenant.ToString() : ColumnMapper.GetTenantMapping().ToString(), group.Key);
var quotaController = storage.QuotaController;
storage.SetQuotaController(null);
try
{
var adjustedPath = file.Path;
var module = ModuleProvider.GetByStorageModule(file.Module, file.Domain);
if (module == null || module.TryAdjustFilePath(Dump, ColumnMapper, ref adjustedPath))
{
var key = file.GetZipKey();
if (Dump)
{
key = CrossPlatform.PathCombine(KeyHelper.GetStorage(), key);
}
using var stream = dataReader.GetEntry(key);
try
{
storage.Save(file.Domain, adjustedPath, module != null ? module.PrepareData(key, stream, ColumnMapper) : stream);
}
catch (Exception error)
{
Logger.WarnFormat("can't restore file ({0}:{1}): {2}", file.Module, file.Path, error);
}
}
}
finally
{
if (quotaController != null)
{
storage.SetQuotaController(quotaController);
}
}
}
SetCurrentStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
}
if (fileGroups.Count == 0)
{
SetStepCompleted();
}
Logger.Debug("end restore storage");
}
public override void RunJob()
{
Logger.Debug("begin restore portal");
Logger.Debug("begin restore data");
using (var dataReader = new ZipReadOperator(BackupFilePath))
{
using (var entry = dataReader.GetEntry(KeyHelper.GetDumpKey()))
{
Dump = entry != null && CoreBaseSettings.Standalone;
}
if (Dump)
{
RestoreFromDump(dataReader);
}
else
{
var modulesToProcess = GetModulesToProcess().ToList();
SetStepsCount(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count);
foreach (var module in modulesToProcess)
{
var restoreTask = new RestoreDbModuleTask(Options, module, dataReader, ColumnMapper, DbFactory, ReplaceDate, Dump, StorageFactory, StorageFactoryConfig, ModuleProvider);
restoreTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress);
foreach (var tableName in IgnoredTables)
{
restoreTask.IgnoreTable(tableName);
}
restoreTask.RunJob();
}
}
Logger.Debug("end restore data");
if (ProcessStorage)
{
if (CoreBaseSettings.Standalone)
{
Logger.Debug("clear cache");
AscCacheNotify.ClearCache();
}
DoRestoreStorage(dataReader);
}
if (UnblockPortalAfterCompleted)
{
SetTenantActive(ColumnMapper.GetTenantMapping());
}
}
if (CoreBaseSettings.Standalone)
{
Logger.Debug("refresh license");
try
{
LicenseReader.RejectLicense();
}
catch (Exception ex)
{
Logger.Error(ex);
}
Logger.Debug("clear cache");
AscCacheNotify.ClearCache();
}
Logger.Debug("end restore portal");
}
public void Init(string toConfigPath, string fromFilePath, int tenantId = -1, ColumnMapper columnMapper = null, string upgradesPath = null)
{
if (fromFilePath == null)
{
throw new ArgumentNullException("fromFilePath");
}
if (!File.Exists(fromFilePath))
{
throw new FileNotFoundException("file not found at given path");
}
BackupFilePath = fromFilePath;
UpgradesPath = upgradesPath;
ColumnMapper = columnMapper ?? new ColumnMapper();
Init(tenantId, toConfigPath);
}
public RestorePortalTask(ILog logger, int tenantId, string toConfigPath, string fromFilePath, ColumnMapper columnMapper = null)
: base(logger, tenantId, toConfigPath)
{
if (fromFilePath == null)
{
throw new ArgumentNullException("fromFilePath");
}
if (!File.Exists(fromFilePath))
{
throw new FileNotFoundException("file not found at given path");
}
BackupFilePath = fromFilePath;
_columnMapper = columnMapper ?? new ColumnMapper();
}
public RestorePortalTask(ILog logger, string toConfigPath, string fromFilePath, ColumnMapper columnMapper = null)
: this(logger, -1, toConfigPath, fromFilePath, columnMapper)
{
}
public override void Run()
{
InvokeInfo("begin transfer portal ({0})", Tenant.TenantAlias);
string backupFilePath = GetBackupFilePath();
var fromDbFactory = new DbFactory(ConfigPath);
var toDbFactory = new DbFactory(ToConfigPath);
var columnMapper = new ColumnMapper();
try
{
//target db can have error tenant from the previous attempts
SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d"));
if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, TenantStatus.Transfering);
}
InitProgress(ProcessStorage ? 3 : 2);
//save db data to temporary file
var backupTask = new BackupPortalTask(Tenant, ConfigPath, backupFilePath)
{
ProcessStorage = false
};
foreach (var moduleName in IgnoredModules)
{
backupTask.IgnoreModule(moduleName);
}
RunSubtask(backupTask);
//restore db data from temporary file
var restoreTask = new RestorePortalTask(ToConfigPath, backupFilePath, columnMapper)
{
ProcessStorage = false
};
foreach (var moduleName in IgnoredModules)
{
restoreTask.IgnoreModule(moduleName);
}
RunSubtask(restoreTask);
//transfer files
if (ProcessStorage)
{
DoTransferStorage(columnMapper);
}
SaveTenant(toDbFactory, TenantStatus.Active);
if (DeleteOldPortalAfterCompletion)
{
SaveTenant(fromDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_deleted");
}
else if (BlockOldPortalAfterStart)
{
SaveTenant(fromDbFactory, TenantStatus.Active);
}
}
catch
{
SaveTenant(fromDbFactory, TenantStatus.Active);
if (columnMapper.GetTenantMapping() > 0)
{
SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error");
}
throw;
}
finally
{
if (DeleteBackupFileAfterCompletion)
{
File.Delete(backupFilePath);
}
InvokeInfo("end transfer portal ({0})", Tenant.TenantAlias);
}
}
public RestoreDbModuleTask(IOptionsMonitor <ILog> options, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump, StorageFactory storageFactory, StorageFactoryConfig storageFactoryConfig, ModuleProvider moduleProvider)
: base(factory, options, storageFactory, storageFactoryConfig, moduleProvider)
{
Reader = reader ?? throw new ArgumentNullException("reader");
ColumnMapper = columnMapper ?? throw new ArgumentNullException("columnMapper");
DbFactory = factory ?? throw new ArgumentNullException("factory");
Module = module;
ReplaceDate = replaceDate;
Dump = dump;
Init(-1, null);
}
private void RestoreTable(DbConnection connection, TableInfo tableInfo, ref int transactionsCommited, ref int rowsInserted)
{
SetColumns(connection, tableInfo);
using var stream = Reader.GetEntry(KeyHelper.GetTableZipKey(Module, tableInfo.Name));
var lowImportanceRelations = Module
.TableRelations
.Where(
r =>
string.Equals(r.ParentTable, tableInfo.Name, StringComparison.InvariantCultureIgnoreCase))
.Where(r => r.Importance == RelationImportance.Low && !r.IsSelfRelation())
.Select(r => Tuple.Create(r, Module.Tables.Single(t => t.Name == r.ChildTable)))
.ToList();
foreach (
var rows in
GetRows(tableInfo, stream)
.Skip(transactionsCommited * TransactionLength)
.MakeParts(TransactionLength))
{
using var transaction = connection.BeginTransaction();
var rowsSuccess = 0;
foreach (var row in rows)
{
if (ReplaceDate)
{
foreach (var column in tableInfo.DateColumns)
{
ColumnMapper.SetDateMapping(tableInfo.Name, column, row[column.Key]);
}
}
object oldIdValue = null;
object newIdValue = null;
if (tableInfo.HasIdColumn())
{
oldIdValue = row[tableInfo.IdColumn];
newIdValue = ColumnMapper.GetMapping(tableInfo.Name, tableInfo.IdColumn, oldIdValue);
if (newIdValue == null)
{
if (tableInfo.IdType == IdType.Guid)
{
newIdValue = Guid.NewGuid().ToString("D");
}
else if (tableInfo.IdType == IdType.Integer)
{
var command = connection.CreateCommand();
command.CommandText = string.Format("select max({0}) from {1};", tableInfo.IdColumn, tableInfo.Name);
newIdValue = (int)command.WithTimeout(120).ExecuteScalar() + 1;
}
}
if (newIdValue != null)
{
ColumnMapper.SetMapping(tableInfo.Name, tableInfo.IdColumn, oldIdValue,
newIdValue);
}
}
var insertCommand = Module.CreateInsertCommand(Dump, connection, ColumnMapper, tableInfo,
row);
if (insertCommand == null)
{
Logger.WarnFormat("Can't create command to insert row to {0} with values [{1}]", tableInfo,
row);
ColumnMapper.Rollback();
continue;
}
insertCommand.WithTimeout(120).ExecuteNonQuery();
rowsSuccess++;
if (tableInfo.HasIdColumn() && tableInfo.IdType == IdType.Autoincrement)
{
var lastIdCommand = DbFactory.CreateLastInsertIdCommand();
lastIdCommand.Connection = connection;
newIdValue = Convert.ToInt32(lastIdCommand.ExecuteScalar());
ColumnMapper.SetMapping(tableInfo.Name, tableInfo.IdColumn, oldIdValue, newIdValue);
}
ColumnMapper.Commit();
foreach (var relation in lowImportanceRelations)
{
if (!relation.Item2.HasTenantColumn())
{
Logger.WarnFormat(
"Table {0} does not contain tenant id column. Can't apply low importance relations on such tables.",
relation.Item2.Name);
continue;
}
var oldValue = row[relation.Item1.ParentColumn];
var newValue = ColumnMapper.GetMapping(relation.Item1.ParentTable,
relation.Item1.ParentColumn, oldValue);
var command = connection.CreateCommand();
command.CommandText = string.Format("update {0} set {1} = {2} where {1} = {3} and {4} = {5}",
relation.Item1.ChildTable,
relation.Item1.ChildColumn,
newValue is string? "'" + newValue + "'" : newValue,
oldValue is string? "'" + oldValue + "'" : oldValue,
relation.Item2.TenantColumn,
ColumnMapper.GetTenantMapping());
command.WithTimeout(120).ExecuteNonQuery();
}
}
transaction.Commit();
transactionsCommited++;
rowsInserted += rowsSuccess;
}
}
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (columnMapper == null)
{
throw new ArgumentNullException("columnMapper");
}
if (factory == null)
{
throw new ArgumentNullException("factory");
}
_module = module;
_reader = reader;
_columnMapper = columnMapper;
_factory = factory;
}
