public RestorePortalTask(string toConfigPath, string fromFilePath, ColumnMapper columnMapper = null) : base(null, toConfigPath) { if (fromFilePath == null) throw new ArgumentNullException("fromFilePath"); if (!File.Exists(fromFilePath)) throw new FileNotFoundException("file not found at given path"); BackupFilePath = fromFilePath; _columnMapper = columnMapper ?? new ColumnMapper(); }
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory) { if (reader == null) throw new ArgumentNullException("reader"); if (columnMapper == null) throw new ArgumentNullException("columnMapper"); if (factory == null) throw new ArgumentNullException("factory"); _module = module; _reader = reader; _columnMapper = columnMapper; _factory = factory; }
public static ColumnMapper ForRestoreDemoPortal(int tenantID) { var columnMapper = new ColumnMapper(); columnMapper.SetMapping("tenants_tenants", "id", tenantID); //set up for projects: columnMapper.SetMapping("projects_comments", "create_on", DateTime.UtcNow); columnMapper.SetMapping("projects_messages", "create_on", DateTime.UtcNow); columnMapper.SetMapping("projects_messages", "last_modified_on", DateTime.UtcNow); columnMapper.SetMapping("projects_milestones", "create_on", DateTime.UtcNow); columnMapper.SetMapping("projects_milestones", "last_modified_on", DateTime.UtcNow); columnMapper.SetMapping("projects_milestones", "deadline", DateTime.UtcNow.AddDays(7), val => !IsDefaultDateTime(val)); columnMapper.SetMapping("projects_projects", "create_on", DateTime.UtcNow); columnMapper.SetMapping("projects_projects", "last_modified_on", DateTime.UtcNow); columnMapper.SetMapping("projects_project_participant", "created", DateTime.UtcNow); columnMapper.SetMapping("projects_project_participant", "updated", DateTime.UtcNow); columnMapper.SetMapping("projects_report_template", "create_on", DateTime.UtcNow); columnMapper.SetMapping("projects_subtasks", "create_on", DateTime.UtcNow); columnMapper.SetMapping("projects_subtasks", "last_modified_on", DateTime.UtcNow); columnMapper.SetMapping("projects_tags", "create_on", DateTime.UtcNow, val => !IsDefaultDateTime(val)); columnMapper.SetMapping("projects_tags", "last_modified_on", DateTime.UtcNow, val => !IsDefaultDateTime(val)); columnMapper.SetMapping("projects_tasks", "deadline", DateTime.UtcNow.AddDays(7), val => !IsDefaultDateTime(val)); columnMapper.SetMapping("projects_tasks", "create_on", DateTime.UtcNow); columnMapper.SetMapping("projects_tasks", "last_modified_on", DateTime.UtcNow); columnMapper.SetMapping("projects_tasks", "start_date", DateTime.UtcNow); columnMapper.SetMapping("projects_templates", "create_on", DateTime.UtcNow); columnMapper.SetMapping("projects_templates", "last_modified_on", DateTime.UtcNow); columnMapper.SetMapping("projects_time_tracking", "create_on", DateTime.UtcNow); //set up for crm: columnMapper.SetMapping("crm_task", "deadline", DateTime.UtcNow.AddDays(7), val => !IsDefaultDateTime(val)); return columnMapper; }
public override void RunJob() { Logger.Debug("begin transfer {0}", TenantId); var fromDbFactory = new DbFactory(ConfigPath); var toDbFactory = new DbFactory(ToConfigPath); string tenantAlias = GetTenantAlias(fromDbFactory); string backupFilePath = GetBackupFilePath(tenantAlias); var columnMapper = new ColumnMapper(); try { //target db can have error tenant from the previous attempts SaveTenant(toDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d")); if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Transfering); } SetStepsCount(ProcessStorage ? 3 : 2); //save db data to temporary file var backupTask = new BackupPortalTask(Logger, TenantId, ConfigPath, backupFilePath, Limit) { ProcessStorage = false }; backupTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress); foreach (var moduleName in IgnoredModules) { backupTask.IgnoreModule(moduleName); } backupTask.RunJob(); //restore db data from temporary file var restoreTask = new RestorePortalTask(Logger, ToConfigPath, backupFilePath, columnMapper) { ProcessStorage = false }; restoreTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress); foreach (var moduleName in IgnoredModules) { restoreTask.IgnoreModule(moduleName); } restoreTask.RunJob(); //transfer files if (ProcessStorage) { DoTransferStorage(columnMapper); } SaveTenant(toDbFactory, tenantAlias, TenantStatus.Active); if (DeleteOldPortalAfterCompletion) { SaveTenant(fromDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_deleted"); } else if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Active); } } catch { SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Active); if (columnMapper.GetTenantMapping() > 0) { SaveTenant(toDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_error"); } throw; } finally { if (DeleteBackupFileAfterCompletion) { File.Delete(backupFilePath); } Logger.Debug("end transfer {0}", TenantId); } }
public RestoreDbModuleTask(ILog logger, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump) : base(logger, -1, null) { if (reader == null) { throw new ArgumentNullException("reader"); } if (columnMapper == null) { throw new ArgumentNullException("columnMapper"); } if (factory == null) { throw new ArgumentNullException("factory"); } _module = module; _reader = reader; _columnMapper = columnMapper; _factory = factory; _replaceDate = replaceDate; this.dump = dump; }
public override void Run() { InvokeInfo("begin transfer portal ({0})", Tenant.TenantAlias); string backupFilePath = GetBackupFilePath(); var fromDbFactory = new DbFactory(ConfigPath); var toDbFactory = new DbFactory(ToConfigPath); var columnMapper = new ColumnMapper(); try { //target db can have error tenant from the previous attempts SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d")); if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, TenantStatus.Transfering); } InitProgress(ProcessStorage ? 3 : 2); //save db data to temporary file var backupTask = new BackupPortalTask(Tenant, ConfigPath, backupFilePath) {ProcessStorage = false}; foreach (var moduleName in IgnoredModules) { backupTask.IgnoreModule(moduleName); } RunSubtask(backupTask); //restore db data from temporary file var restoreTask = new RestorePortalTask(ToConfigPath, backupFilePath, columnMapper) {ProcessStorage = false}; foreach (var moduleName in IgnoredModules) { restoreTask.IgnoreModule(moduleName); } RunSubtask(restoreTask); //transfer files if (ProcessStorage) { DoTransferStorage(columnMapper); } SaveTenant(toDbFactory, TenantStatus.Active); if (DeleteOldPortalAfterCompletion) { SaveTenant(fromDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_deleted"); } else if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, TenantStatus.Active); } } catch { SaveTenant(fromDbFactory, TenantStatus.Active); if (columnMapper.GetTenantMapping() > 0) { SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error"); } throw; } finally { if (DeleteBackupFileAfterCompletion) { File.Delete(backupFilePath); } InvokeInfo("end transfer portal ({0})", Tenant.TenantAlias); } }
private void DoTransferStorage(ColumnMapper columnMapper) { InvokeInfo("begin transfer storage"); var fileGroups = GetFilesToProcess().GroupBy(file => file.Module).ToList(); int groupsProcessed = 0; foreach (var group in fileGroups) { ICrossModuleTransferUtility transferUtility = StorageFactory.GetCrossModuleTransferUtility( ConfigPath, Tenant.TenantId, group.Key, ToConfigPath, columnMapper.GetTenantMapping(), group.Key); foreach (BackupFileInfo file in group) { string adjustedPath = file.Path; IModuleSpecifics module = ModuleProvider.GetByStorageModule(file.Module, file.Domain); if (module == null || module.TryAdjustFilePath(columnMapper, ref adjustedPath)) { try { transferUtility.CopyFile(file.Domain, file.Path, file.Domain, adjustedPath); } catch (Exception error) { InvokeWarning("Can't copy file ({0}:{1}): {2}", file.Module, file.Path, error); } } else { InvokeWarning("Can't adjust file path \"{0}\".", file.Path); } } SetStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count)); } if (fileGroups.Count == 0) SetStepCompleted(); InvokeInfo("end transfer storage"); }
private void DoRestoreStorage(IDataReadOperator dataReader) { Logger.Debug("begin restore storage"); var fileGroups = GetFilesToProcess(dataReader).GroupBy(file => file.Module).ToList(); var groupsProcessed = 0; foreach (var group in fileGroups) { foreach (var file in group) { var storage = StorageFactory.GetStorage(ConfigPath, Dump ? file.Tenant.ToString() : ColumnMapper.GetTenantMapping().ToString(), group.Key); var quotaController = storage.QuotaController; storage.SetQuotaController(null); try { var adjustedPath = file.Path; var module = ModuleProvider.GetByStorageModule(file.Module, file.Domain); if (module == null || module.TryAdjustFilePath(Dump, ColumnMapper, ref adjustedPath)) { var key = file.GetZipKey(); if (Dump) { key = CrossPlatform.PathCombine(KeyHelper.GetStorage(), key); } using var stream = dataReader.GetEntry(key); try { storage.Save(file.Domain, adjustedPath, module != null ? module.PrepareData(key, stream, ColumnMapper) : stream); } catch (Exception error) { Logger.WarnFormat("can't restore file ({0}:{1}): {2}", file.Module, file.Path, error); } } } finally { if (quotaController != null) { storage.SetQuotaController(quotaController); } } } SetCurrentStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count)); } if (fileGroups.Count == 0) { SetStepCompleted(); } Logger.Debug("end restore storage"); }
public override void RunJob() { Logger.Debug("begin restore portal"); Logger.Debug("begin restore data"); using (var dataReader = new ZipReadOperator(BackupFilePath)) { using (var entry = dataReader.GetEntry(KeyHelper.GetDumpKey())) { Dump = entry != null && CoreBaseSettings.Standalone; } if (Dump) { RestoreFromDump(dataReader); } else { var modulesToProcess = GetModulesToProcess().ToList(); SetStepsCount(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count); foreach (var module in modulesToProcess) { var restoreTask = new RestoreDbModuleTask(Options, module, dataReader, ColumnMapper, DbFactory, ReplaceDate, Dump, StorageFactory, StorageFactoryConfig, ModuleProvider); restoreTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress); foreach (var tableName in IgnoredTables) { restoreTask.IgnoreTable(tableName); } restoreTask.RunJob(); } } Logger.Debug("end restore data"); if (ProcessStorage) { if (CoreBaseSettings.Standalone) { Logger.Debug("clear cache"); AscCacheNotify.ClearCache(); } DoRestoreStorage(dataReader); } if (UnblockPortalAfterCompleted) { SetTenantActive(ColumnMapper.GetTenantMapping()); } } if (CoreBaseSettings.Standalone) { Logger.Debug("refresh license"); try { LicenseReader.RejectLicense(); } catch (Exception ex) { Logger.Error(ex); } Logger.Debug("clear cache"); AscCacheNotify.ClearCache(); } Logger.Debug("end restore portal"); }
public void Init(string toConfigPath, string fromFilePath, int tenantId = -1, ColumnMapper columnMapper = null, string upgradesPath = null) { if (fromFilePath == null) { throw new ArgumentNullException("fromFilePath"); } if (!File.Exists(fromFilePath)) { throw new FileNotFoundException("file not found at given path"); } BackupFilePath = fromFilePath; UpgradesPath = upgradesPath; ColumnMapper = columnMapper ?? new ColumnMapper(); Init(tenantId, toConfigPath); }
public RestorePortalTask(ILog logger, int tenantId, string toConfigPath, string fromFilePath, ColumnMapper columnMapper = null) : base(logger, tenantId, toConfigPath) { if (fromFilePath == null) { throw new ArgumentNullException("fromFilePath"); } if (!File.Exists(fromFilePath)) { throw new FileNotFoundException("file not found at given path"); } BackupFilePath = fromFilePath; _columnMapper = columnMapper ?? new ColumnMapper(); }
public RestorePortalTask(ILog logger, string toConfigPath, string fromFilePath, ColumnMapper columnMapper = null) : this(logger, -1, toConfigPath, fromFilePath, columnMapper) { }
public override void Run() { InvokeInfo("begin transfer portal ({0})", Tenant.TenantAlias); string backupFilePath = GetBackupFilePath(); var fromDbFactory = new DbFactory(ConfigPath); var toDbFactory = new DbFactory(ToConfigPath); var columnMapper = new ColumnMapper(); try { //target db can have error tenant from the previous attempts SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d")); if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, TenantStatus.Transfering); } InitProgress(ProcessStorage ? 3 : 2); //save db data to temporary file var backupTask = new BackupPortalTask(Tenant, ConfigPath, backupFilePath) { ProcessStorage = false }; foreach (var moduleName in IgnoredModules) { backupTask.IgnoreModule(moduleName); } RunSubtask(backupTask); //restore db data from temporary file var restoreTask = new RestorePortalTask(ToConfigPath, backupFilePath, columnMapper) { ProcessStorage = false }; foreach (var moduleName in IgnoredModules) { restoreTask.IgnoreModule(moduleName); } RunSubtask(restoreTask); //transfer files if (ProcessStorage) { DoTransferStorage(columnMapper); } SaveTenant(toDbFactory, TenantStatus.Active); if (DeleteOldPortalAfterCompletion) { SaveTenant(fromDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_deleted"); } else if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, TenantStatus.Active); } } catch { SaveTenant(fromDbFactory, TenantStatus.Active); if (columnMapper.GetTenantMapping() > 0) { SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error"); } throw; } finally { if (DeleteBackupFileAfterCompletion) { File.Delete(backupFilePath); } InvokeInfo("end transfer portal ({0})", Tenant.TenantAlias); } }
public RestoreDbModuleTask(IOptionsMonitor <ILog> options, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump, StorageFactory storageFactory, StorageFactoryConfig storageFactoryConfig, ModuleProvider moduleProvider) : base(factory, options, storageFactory, storageFactoryConfig, moduleProvider) { Reader = reader ?? throw new ArgumentNullException("reader"); ColumnMapper = columnMapper ?? throw new ArgumentNullException("columnMapper"); DbFactory = factory ?? throw new ArgumentNullException("factory"); Module = module; ReplaceDate = replaceDate; Dump = dump; Init(-1, null); }
private void RestoreTable(DbConnection connection, TableInfo tableInfo, ref int transactionsCommited, ref int rowsInserted) { SetColumns(connection, tableInfo); using var stream = Reader.GetEntry(KeyHelper.GetTableZipKey(Module, tableInfo.Name)); var lowImportanceRelations = Module .TableRelations .Where( r => string.Equals(r.ParentTable, tableInfo.Name, StringComparison.InvariantCultureIgnoreCase)) .Where(r => r.Importance == RelationImportance.Low && !r.IsSelfRelation()) .Select(r => Tuple.Create(r, Module.Tables.Single(t => t.Name == r.ChildTable))) .ToList(); foreach ( var rows in GetRows(tableInfo, stream) .Skip(transactionsCommited * TransactionLength) .MakeParts(TransactionLength)) { using var transaction = connection.BeginTransaction(); var rowsSuccess = 0; foreach (var row in rows) { if (ReplaceDate) { foreach (var column in tableInfo.DateColumns) { ColumnMapper.SetDateMapping(tableInfo.Name, column, row[column.Key]); } } object oldIdValue = null; object newIdValue = null; if (tableInfo.HasIdColumn()) { oldIdValue = row[tableInfo.IdColumn]; newIdValue = ColumnMapper.GetMapping(tableInfo.Name, tableInfo.IdColumn, oldIdValue); if (newIdValue == null) { if (tableInfo.IdType == IdType.Guid) { newIdValue = Guid.NewGuid().ToString("D"); } else if (tableInfo.IdType == IdType.Integer) { var command = connection.CreateCommand(); command.CommandText = string.Format("select max({0}) from {1};", tableInfo.IdColumn, tableInfo.Name); newIdValue = (int)command.WithTimeout(120).ExecuteScalar() + 1; } } if (newIdValue != null) { ColumnMapper.SetMapping(tableInfo.Name, tableInfo.IdColumn, oldIdValue, newIdValue); } } var insertCommand = Module.CreateInsertCommand(Dump, connection, ColumnMapper, tableInfo, row); if (insertCommand == null) { Logger.WarnFormat("Can't create command to insert row to {0} with values [{1}]", tableInfo, row); ColumnMapper.Rollback(); continue; } insertCommand.WithTimeout(120).ExecuteNonQuery(); rowsSuccess++; if (tableInfo.HasIdColumn() && tableInfo.IdType == IdType.Autoincrement) { var lastIdCommand = DbFactory.CreateLastInsertIdCommand(); lastIdCommand.Connection = connection; newIdValue = Convert.ToInt32(lastIdCommand.ExecuteScalar()); ColumnMapper.SetMapping(tableInfo.Name, tableInfo.IdColumn, oldIdValue, newIdValue); } ColumnMapper.Commit(); foreach (var relation in lowImportanceRelations) { if (!relation.Item2.HasTenantColumn()) { Logger.WarnFormat( "Table {0} does not contain tenant id column. Can't apply low importance relations on such tables.", relation.Item2.Name); continue; } var oldValue = row[relation.Item1.ParentColumn]; var newValue = ColumnMapper.GetMapping(relation.Item1.ParentTable, relation.Item1.ParentColumn, oldValue); var command = connection.CreateCommand(); command.CommandText = string.Format("update {0} set {1} = {2} where {1} = {3} and {4} = {5}", relation.Item1.ChildTable, relation.Item1.ChildColumn, newValue is string? "'" + newValue + "'" : newValue, oldValue is string? "'" + oldValue + "'" : oldValue, relation.Item2.TenantColumn, ColumnMapper.GetTenantMapping()); command.WithTimeout(120).ExecuteNonQuery(); } } transaction.Commit(); transactionsCommited++; rowsInserted += rowsSuccess; } }
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory) { if (reader == null) { throw new ArgumentNullException("reader"); } if (columnMapper == null) { throw new ArgumentNullException("columnMapper"); } if (factory == null) { throw new ArgumentNullException("factory"); } _module = module; _reader = reader; _columnMapper = columnMapper; _factory = factory; }