public override void Run() { InvokeInfo("begin restore portal"); List<IModuleSpecifics> modulesToProcess = GetModulesToProcess().ToList(); InitProgress(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count); using (var dataReader = new ZipReadOperator(BackupFilePath)) { InvokeInfo("begin restore data"); var dbFactory = new DbFactory(ConfigPath); foreach (var module in modulesToProcess) { var restoreTask = new RestoreDbModuleTask(module, dataReader, _columnMapper, dbFactory, ReplaceDate); foreach (var tableName in IgnoredTables) { restoreTask.IgnoreTable(tableName); } RunSubtask(restoreTask); } InvokeInfo("end restore data"); if (ProcessStorage) DoRestoreStorage(dataReader); if (UnblockPortalAfterCompleted) SetTenantActive(dbFactory, _columnMapper.GetTenantMapping()); } InvokeInfo("end restore portal"); }
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory) { if (reader == null) throw new ArgumentNullException("reader"); if (columnMapper == null) throw new ArgumentNullException("columnMapper"); if (factory == null) throw new ArgumentNullException("factory"); _module = module; _reader = reader; _columnMapper = columnMapper; _factory = factory; }
public override void Run() { InvokeInfo("begin delete portal ({0})", Tenant.TenantAlias); List<IModuleSpecifics> modulesToProcess = GetModulesToProcess().Reverse().ToList(); InitProgress(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count); var dbFactory = new DbFactory(ConfigPath); foreach (var module in modulesToProcess) { DoDeleteModule(dbFactory, module); } if (ProcessStorage) { DoDeleteStorage(); } InvokeInfo("end delete portal ({0})", Tenant.TenantAlias); }
private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module) { InvokeInfo("begin saving data for module {0}", module.ModuleName); int tablesCount = module.Tables.Count(); int tablesProcessed = 0; using (var connection = dbFactory.OpenConnection(module.ConnectionStringName)) { foreach (var table in module.Tables) { InvokeInfo("begin saving table {0}", table.Name); using (var data = new DataTable(table.Name)) { ActionInvoker.Try( state => { data.Clear(); var t = (TableInfo)state; var dataAdapter = dbFactory.CreateDataAdapter(module.ConnectionStringName); dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(600); ((DbDataAdapter)dataAdapter).Fill(data); }, table, maxAttempts: 5, onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); }, onAttemptFailure: error => InvokeWarning("backup attempt failure: {0}", error)); foreach (var col in data.Columns.Cast<DataColumn>().Where(col => col.DataType == typeof(DateTime))) { col.DateTimeMode = DataSetDateTime.Unspecified; } var stream = writer.BeginWriteEntry(KeyHelper.GetTableZipKey(module, data.TableName)); data.WriteXml(stream, XmlWriteMode.WriteSchema); writer.EndWriteEntry(); data.Clear(); } SetStepProgress((int)((++tablesProcessed*100)/(double)tablesCount)); } } InvokeInfo("end saving data for module {0}", module.ModuleName); }
private void DoDeleteModule(DbFactory dbFactory, IModuleSpecifics module) { InvokeInfo("begin delete data for module ({0})", module.ModuleName); int tablesCount = module.Tables.Count(); int tablesProcessed = 0; using (var connection = dbFactory.OpenConnection(module.ConnectionStringName)) { foreach (var table in module.GetTablesOrdered().Reverse().Where(t => !IgnoredTables.Contains(t.Name))) { ActionInvoker.Try(state => { var t = (TableInfo)state; module.CreateDeleteCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(120).ExecuteNonQuery(); }, table, 5, onFailure: error => { throw ThrowHelper.CantDeleteTable(table.Name, error); }); SetStepProgress((int)((++tablesProcessed*100)/(double)tablesCount)); } } InvokeInfo("end delete data for module ({0})", module.ModuleName); }
public override void Run() { InvokeInfo("begin backup portal ({0})", Tenant.TenantAlias); List<IModuleSpecifics> modulesToProcess = GetModulesToProcess().ToList(); InitProgress(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count); using (var writer = new ZipWriteOperator(BackupFilePath)) { var dbFactory = new DbFactory(ConfigPath); foreach (var module in modulesToProcess) { DoBackupModule(writer, dbFactory, module); } if (ProcessStorage) { DoBackupStorage(writer); } } InvokeInfo("end backup portal ({0})", Tenant.TenantAlias); }
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory) { if (reader == null) { throw new ArgumentNullException("reader"); } if (columnMapper == null) { throw new ArgumentNullException("columnMapper"); } if (factory == null) { throw new ArgumentNullException("factory"); } _module = module; _reader = reader; _columnMapper = columnMapper; _factory = factory; }
public override void RunJob() { Logger.Debug("begin restore portal"); Logger.Debug("begin restore data"); using (var dataReader = new ZipReadOperator(BackupFilePath)) { using (var entry = dataReader.GetEntry(KeyHelper.GetDumpKey())) { Dump = entry != null && CoreContext.Configuration.Standalone; } var dbFactory = new DbFactory(ConfigPath); if (Dump) { RestoreFromDump(dataReader); } else { var modulesToProcess = GetModulesToProcess().ToList(); SetStepsCount(ProcessStorage ? modulesToProcess.Count + 1 : modulesToProcess.Count); foreach (var module in modulesToProcess) { var restoreTask = new RestoreDbModuleTask(Logger, module, dataReader, _columnMapper, dbFactory, ReplaceDate, Dump); restoreTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress); foreach (var tableName in IgnoredTables) { restoreTask.IgnoreTable(tableName); } restoreTask.RunJob(); } } Logger.Debug("end restore data"); if (ProcessStorage) { if (CoreContext.Configuration.Standalone) { Logger.Debug("clear cache"); AscCache.ClearCache(); } DoRestoreStorage(dataReader); } if (UnblockPortalAfterCompleted) { SetTenantActive(dbFactory, _columnMapper.GetTenantMapping()); } } if (CoreContext.Configuration.Standalone) { Logger.Debug("refresh license"); try { LicenseReader.RejectLicense(); } catch (Exception ex) { Logger.Error(ex); } Logger.Debug("clear cache"); AscCache.ClearCache(); } Logger.Debug("end restore portal"); }
public override void Run() { InvokeInfo("begin transfer portal ({0})", Tenant.TenantAlias); string backupFilePath = GetBackupFilePath(); var fromDbFactory = new DbFactory(ConfigPath); var toDbFactory = new DbFactory(ToConfigPath); var columnMapper = new ColumnMapper(); try { //target db can have error tenant from the previous attempts SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d")); if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, TenantStatus.Transfering); } InitProgress(ProcessStorage ? 3 : 2); //save db data to temporary file var backupTask = new BackupPortalTask(Tenant, ConfigPath, backupFilePath) { ProcessStorage = false }; foreach (var moduleName in IgnoredModules) { backupTask.IgnoreModule(moduleName); } RunSubtask(backupTask); //restore db data from temporary file var restoreTask = new RestorePortalTask(ToConfigPath, backupFilePath, columnMapper) { ProcessStorage = false }; foreach (var moduleName in IgnoredModules) { restoreTask.IgnoreModule(moduleName); } RunSubtask(restoreTask); //transfer files if (ProcessStorage) { DoTransferStorage(columnMapper); } SaveTenant(toDbFactory, TenantStatus.Active); if (DeleteOldPortalAfterCompletion) { SaveTenant(fromDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_deleted"); } else if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, TenantStatus.Active); } } catch { SaveTenant(fromDbFactory, TenantStatus.Active); if (columnMapper.GetTenantMapping() > 0) { SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error"); } throw; } finally { if (DeleteBackupFileAfterCompletion) { File.Delete(backupFilePath); } InvokeInfo("end transfer portal ({0})", Tenant.TenantAlias); } }
private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module) { Logger.DebugFormat("begin saving data for module {0}", module.ModuleName); var tablesToProcess = module.Tables.Where(t => !IgnoredTables.Contains(t.Name) && t.InsertMethod != InsertMethod.None).ToList(); var tablesCount = tablesToProcess.Count; var tablesProcessed = 0; using (var connection = dbFactory.OpenConnection()) { foreach (var table in tablesToProcess) { Logger.DebugFormat("begin load table {0}", table.Name); using (var data = new DataTable(table.Name)) { ActionInvoker.Try( state => { data.Clear(); int counts; var offset = 0; do { var t = (TableInfo)state; var dataAdapter = dbFactory.CreateDataAdapter(); dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), TenantId, t, Limit, offset).WithTimeout(600); counts = ((DbDataAdapter)dataAdapter).Fill(data); offset += Limit; } while (counts == Limit); }, table, maxAttempts: 5, onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); }, onAttemptFailure: error => Logger.Warn("backup attempt failure: {0}", error)); foreach (var col in data.Columns.Cast <DataColumn>().Where(col => col.DataType == typeof(DateTime))) { col.DateTimeMode = DataSetDateTime.Unspecified; } module.PrepareData(data); Logger.DebugFormat("end load table {0}", table.Name); Logger.DebugFormat("begin saving table {0}", table.Name); var tmp = Path.GetTempFileName(); using (var file = File.OpenWrite(tmp)) { data.WriteXml(file, XmlWriteMode.WriteSchema); data.Clear(); } writer.WriteEntry(KeyHelper.GetTableZipKey(module, data.TableName), tmp); File.Delete(tmp); Logger.DebugFormat("end saving table {0}", table.Name); } SetCurrentStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount)); } } Logger.DebugFormat("end saving data for module {0}", module.ModuleName); }
public override void RunJob() { Logger.DebugFormat("begin transfer {0}", TenantId); var fromDbFactory = new DbFactory(null, null); var toDbFactory = new DbFactory(null, null); var tenantAlias = GetTenantAlias(fromDbFactory); var backupFilePath = GetBackupFilePath(tenantAlias); var columnMapper = new ColumnMapper(); try { //target db can have error tenant from the previous attempts SaveTenant(toDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d")); if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Transfering); } SetStepsCount(ProcessStorage ? 3 : 2); //save db data to temporary file var backupTask = ServiceProvider.GetService <BackupPortalTask>(); backupTask.Init(TenantId, ConfigPath, backupFilePath, Limit); backupTask.ProcessStorage = false; backupTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress); foreach (var moduleName in IgnoredModules) { backupTask.IgnoreModule(moduleName); } backupTask.RunJob(); //restore db data from temporary file var restoreTask = ServiceProvider.GetService <RestorePortalTask>(); restoreTask.Init(ToConfigPath, backupFilePath, columnMapper: columnMapper); restoreTask.ProcessStorage = false; restoreTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress); foreach (var moduleName in IgnoredModules) { restoreTask.IgnoreModule(moduleName); } restoreTask.RunJob(); //transfer files if (ProcessStorage) { DoTransferStorage(columnMapper); } SaveTenant(toDbFactory, tenantAlias, TenantStatus.Active); if (DeleteOldPortalAfterCompletion) { SaveTenant(fromDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_deleted"); } else if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Active); } } catch { SaveTenant(fromDbFactory, tenantAlias, TenantStatus.Active); if (columnMapper.GetTenantMapping() > 0) { SaveTenant(toDbFactory, tenantAlias, TenantStatus.RemovePending, tenantAlias + "_error"); } throw; } finally { if (DeleteBackupFileAfterCompletion) { File.Delete(backupFilePath); } Logger.DebugFormat("end transfer {0}", TenantId); } }
public DeletePortalTask(DbFactory dbFactory, IOptionsMonitor <ILog> options, int tenantId, string configPath, StorageFactory storageFactory, StorageFactoryConfig storageFactoryConfig, ModuleProvider moduleProvider) : base(dbFactory, options, storageFactory, storageFactoryConfig, moduleProvider) { Init(tenantId, configPath); }
public RestoreDbModuleTask(ILog logger, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump) : base(logger, -1, null) { if (reader == null) { throw new ArgumentNullException("reader"); } if (columnMapper == null) { throw new ArgumentNullException("columnMapper"); } if (factory == null) { throw new ArgumentNullException("factory"); } _module = module; _reader = reader; _columnMapper = columnMapper; _factory = factory; _replaceDate = replaceDate; this.dump = dump; }
private static void SetTenantActive(DbFactory dbFactory, int tenantId) { using (var connection = dbFactory.OpenConnection("core")) { var commandText = string.Format( "update tenants_tenants " + "set " + " status={0}, " + " last_modified='{1}', " + " statuschanged='{1}' " + "where id = '{2}'", (int)TenantStatus.Active, DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss"), tenantId); connection.CreateCommand(commandText).WithTimeout(120).ExecuteNonQuery(); } }
private void DumpTableData(string t, string dir, int count) { try { if (count == 0) { Logger.DebugFormat("dump table data stop {0}", t); SetStepCompleted(2); return; } Logger.DebugFormat("dump table data start {0}", t); var searchWithPrimary = false; string primaryIndex; var primaryIndexStep = 0; var primaryIndexStart = 0; List <string> columns; using (var connection = DbFactory.OpenConnection()) { var command = connection.CreateCommand(); command.CommandText = string.Format("SHOW COLUMNS FROM `{0}`", t); columns = ExecuteList(command).Select(r => "`" + Convert.ToString(r[0]) + "`").ToList(); if (command.CommandText.Contains("tenants_quota") || command.CommandText.Contains("webstudio_settings")) { } } using (var connection = DbFactory.OpenConnection()) { var command = connection.CreateCommand(); command.CommandText = string.Format("select COLUMN_NAME from information_schema.`COLUMNS` where TABLE_SCHEMA = '{0}' and TABLE_NAME = '{1}' and COLUMN_KEY = 'PRI' and DATA_TYPE = 'int'", connection.Database, t); primaryIndex = ExecuteList(command).ConvertAll(r => Convert.ToString(r[0])).FirstOrDefault(); } using (var connection = DbFactory.OpenConnection()) { var command = connection.CreateCommand(); command.CommandText = string.Format("SHOW INDEXES FROM {0} WHERE COLUMN_NAME='{1}' AND seq_in_index=1", t, primaryIndex); var isLeft = ExecuteList(command); searchWithPrimary = isLeft.Count == 1; } if (searchWithPrimary) { using var connection = DbFactory.OpenConnection(); var command = connection.CreateCommand(); command.CommandText = string.Format("select max({1}), min({1}) from {0}", t, primaryIndex); var minMax = ExecuteList(command).ConvertAll(r => new Tuple <int, int>(Convert.ToInt32(r[0]), Convert.ToInt32(r[1]))).FirstOrDefault(); primaryIndexStart = minMax.Item2; primaryIndexStep = (minMax.Item1 - minMax.Item2) / count; if (primaryIndexStep < Limit) { primaryIndexStep = Limit; } } var path = Path.Combine(dir, t); var offset = 0; do { List <object[]> result; if (searchWithPrimary) { result = GetDataWithPrimary(t, columns, primaryIndex, primaryIndexStart, primaryIndexStep); primaryIndexStart += primaryIndexStep; } else { result = GetData(t, columns, offset); } offset += Limit; var resultCount = result.Count; if (resultCount == 0) { break; } SaveToFile(path, t, columns, result); if (resultCount < Limit) { break; } } while (true); SetStepCompleted(); Logger.DebugFormat("dump table data stop {0}", t); } catch (Exception e) { Logger.Error(e); throw; } }
private void DoDump(IDataWriteOperator writer) { var tmp = Path.GetTempFileName(); File.AppendAllText(tmp, true.ToString()); writer.WriteEntry(KeyHelper.GetDumpKey(), tmp); List <string> tables; var files = new List <BackupFileInfo>(); using (var connection = DbFactory.OpenConnection()) { var command = connection.CreateCommand(); command.CommandText = "show tables"; tables = ExecuteList(command).Select(r => Convert.ToString(r[0])).ToList(); } /* using (var dbManager = new DbManager("default", 100000)) * { * tables = dbManager.ExecuteList("show tables;").Select(r => Convert.ToString(r[0])).ToList(); * }*/ var stepscount = tables.Count * 4; // (schema + data) * (dump + zip) if (ProcessStorage) { var tenants = TenantManager.GetTenants(false).Select(r => r.TenantId); foreach (var t in tenants) { files.AddRange(GetFiles(t)); } stepscount += files.Count * 2 + 1; Logger.Debug("files:" + files.Count); } SetStepsCount(stepscount); var excluded = ModuleProvider.AllModules.Where(r => IgnoredModules.Contains(r.ModuleName)).SelectMany(r => r.Tables).Select(r => r.Name).ToList(); excluded.AddRange(IgnoredTables); excluded.Add("res_"); var dir = Path.GetDirectoryName(BackupFilePath); var subDir = Path.Combine(dir, Path.GetFileNameWithoutExtension(BackupFilePath)); var schemeDir = Path.Combine(subDir, KeyHelper.GetDatabaseSchema()); var dataDir = Path.Combine(subDir, KeyHelper.GetDatabaseData()); if (!Directory.Exists(schemeDir)) { Directory.CreateDirectory(schemeDir); } if (!Directory.Exists(dataDir)) { Directory.CreateDirectory(dataDir); } var dict = tables.ToDictionary(t => t, SelectCount); tables.Sort((pair1, pair2) => dict[pair1].CompareTo(dict[pair2])); for (var i = 0; i < tables.Count; i += TasksLimit) { var tasks = new List <Task>(TasksLimit * 2); for (var j = 0; j < TasksLimit && i + j < tables.Count; j++) { var t = tables[i + j]; tasks.Add(Task.Run(() => DumpTableScheme(t, schemeDir))); if (!excluded.Any(t.StartsWith)) { tasks.Add(Task.Run(() => DumpTableData(t, dataDir, dict[t]))); } else { SetStepCompleted(2); } } Task.WaitAll(tasks.ToArray()); ArchiveDir(writer, subDir); } Logger.DebugFormat("dir remove start {0}", subDir); Directory.Delete(subDir, true); Logger.DebugFormat("dir remove end {0}", subDir); if (ProcessStorage) { DoDumpStorage(writer, files); } }
public override void Run() { InvokeInfo("begin transfer portal ({0})", Tenant.TenantAlias); string backupFilePath = GetBackupFilePath(); var fromDbFactory = new DbFactory(ConfigPath); var toDbFactory = new DbFactory(ToConfigPath); var columnMapper = new ColumnMapper(); try { //target db can have error tenant from the previous attempts SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error", "status = " + TenantStatus.Restoring.ToString("d")); if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, TenantStatus.Transfering); } InitProgress(ProcessStorage ? 3 : 2); //save db data to temporary file var backupTask = new BackupPortalTask(Tenant, ConfigPath, backupFilePath) {ProcessStorage = false}; foreach (var moduleName in IgnoredModules) { backupTask.IgnoreModule(moduleName); } RunSubtask(backupTask); //restore db data from temporary file var restoreTask = new RestorePortalTask(ToConfigPath, backupFilePath, columnMapper) {ProcessStorage = false}; foreach (var moduleName in IgnoredModules) { restoreTask.IgnoreModule(moduleName); } RunSubtask(restoreTask); //transfer files if (ProcessStorage) { DoTransferStorage(columnMapper); } SaveTenant(toDbFactory, TenantStatus.Active); if (DeleteOldPortalAfterCompletion) { SaveTenant(fromDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_deleted"); } else if (BlockOldPortalAfterStart) { SaveTenant(fromDbFactory, TenantStatus.Active); } } catch { SaveTenant(fromDbFactory, TenantStatus.Active); if (columnMapper.GetTenantMapping() > 0) { SaveTenant(toDbFactory, TenantStatus.RemovePending, Tenant.TenantAlias + "_error"); } throw; } finally { if (DeleteBackupFileAfterCompletion) { File.Delete(backupFilePath); } InvokeInfo("end transfer portal ({0})", Tenant.TenantAlias); } }
private void SaveTenant(DbFactory dbFactory, TenantStatus status, string newAlias = null, string whereCondition = null) { using (var connection = dbFactory.OpenConnection("core")) { if (newAlias == null) { newAlias = Tenant.TenantAlias; } else if (newAlias != Tenant.TenantAlias) { newAlias = GetUniqAlias(connection, newAlias); } var commandText = string.Format( "update tenants_tenants " + "set " + " status={0}, " + " alias = '{1}', " + " last_modified='{2}', " + " statuschanged='{2}' " + "where alias = '{3}'", status.ToString("d"), newAlias, DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss"), Tenant.TenantAlias); if (!string.IsNullOrEmpty(whereCondition)) commandText += (" and " + whereCondition); connection.CreateCommand(commandText).WithTimeout(120).ExecuteNonQuery(); } }