public RestoreDbModuleTask(ILog logger, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump) : base(logger, -1, null) { if (reader == null) { throw new ArgumentNullException("reader"); } if (columnMapper == null) { throw new ArgumentNullException("columnMapper"); } if (factory == null) { throw new ArgumentNullException("factory"); } _module = module; _reader = reader; _columnMapper = columnMapper; _factory = factory; _replaceDate = replaceDate; this.dump = dump; }
private void DoRestoreStorage(IDataReadOperator dataReader) { InvokeInfo("begin restore storage"); var fileGroups = GetFilesToProcess(dataReader).GroupBy(file => file.Module).ToList(); int groupsProcessed = 0; foreach (var group in fileGroups) { IDataStore storage = StorageFactory.GetStorage(ConfigPath, _columnMapper.GetTenantMapping().ToString(), group.Key, null, null); foreach (BackupFileInfo file in group) { string adjustedPath = file.Path; IModuleSpecifics module = ModuleProvider.GetByStorageModule(file.Module); if (module == null || module.TryAdjustFilePath(_columnMapper, ref adjustedPath)) { Stream stream = dataReader.GetEntry(KeyHelper.GetFileZipKey(file)); try { storage.Save(file.Domain, adjustedPath, stream); } catch (Exception error) { InvokeWarning("can't restore file ({0}:{1}): {2}", file.Module, file.Path, error); } } } SetStepProgress((int)(++groupsProcessed*100/(double)fileGroups.Count)); } if (fileGroups.Count == 0) SetStepCompleted(); InvokeInfo("end restore storage"); }
public void LoadFrom(IEnumerable<XElement> elements, int tenant, string[] configs, IDataReadOperator reader) { processedTables.Clear(); foreach (var connectionString in GetConnectionStrings(configs)) { RestoreDatabase(connectionString, elements, reader); } }
private async Task RestoreFromDumpFile(IDataReadOperator dataReader, string fileName) { Logger.DebugFormat("Restore from {0}", fileName); using (var stream = dataReader.GetEntry(fileName)) { await RunMysqlFile(stream); } SetStepCompleted(); }
private void DoRestoreStorage(IDataReadOperator dataReader) { Logger.Debug("begin restore storage"); var fileGroups = GetFilesToProcess(dataReader).GroupBy(file => file.Module).ToList(); var groupsProcessed = 0; foreach (var group in fileGroups) { foreach (var file in group) { var storage = StorageFactory.GetStorage(ConfigPath, Dump ? file.Tenant.ToString() : _columnMapper.GetTenantMapping().ToString(), group.Key); var quotaController = storage.QuotaController; storage.SetQuotaController(null); try { var adjustedPath = file.Path; var module = ModuleProvider.GetByStorageModule(file.Module, file.Domain); if (module == null || module.TryAdjustFilePath(Dump, _columnMapper, ref adjustedPath)) { var key = file.GetZipKey(); if (Dump) { key = Path.Combine(KeyHelper.GetStorage(), key); } using (var stream = dataReader.GetEntry(key)) { try { storage.Save(file.Domain, adjustedPath, module != null ? module.PrepareData(key, stream, _columnMapper) : stream); } catch (Exception error) { Logger.WarnFormat("can't restore file ({0}:{1}): {2}", file.Module, file.Path, error); } } } } finally { if (quotaController != null) { storage.SetQuotaController(quotaController); } } } SetCurrentStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count)); } if (fileGroups.Count == 0) { SetStepCompleted(); } Logger.Debug("end restore storage"); }
private IEnumerable <BackupFileInfo> GetFilesToProcess(IDataReadOperator dataReader) { using var stream = dataReader.GetEntry(KeyHelper.GetStorageRestoreInfoZipKey()); if (stream == null) { return(Enumerable.Empty <BackupFileInfo>()); } var restoreInfo = XElement.Load(new StreamReader(stream)); return(restoreInfo.Elements("file").Select(BackupFileInfo.FromXElement).ToList()); }
private void RestoreFromDump(IDataReadOperator dataReader) { var keyBase = KeyHelper.GetDatabaseSchema(); var keys = dataReader.GetEntries(keyBase).Select(r => Path.GetFileName(r)).ToList(); var upgrades = new List <string>(); if (!string.IsNullOrEmpty(UpgradesPath) && Directory.Exists(UpgradesPath)) { upgrades = Directory.GetFiles(UpgradesPath).ToList(); } var stepscount = keys.Count * 2 + upgrades.Count; SetStepsCount(ProcessStorage ? stepscount + 1 : stepscount); if (ProcessStorage) { var storageModules = StorageFactoryConfig.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed); var tenants = TenantManager.GetTenants(false); stepscount += storageModules.Count() * tenants.Count; SetStepsCount(stepscount + 1); DoDeleteStorage(storageModules, tenants); } else { SetStepsCount(stepscount); } for (var i = 0; i < keys.Count; i += TasksLimit) { var tasks = new List <Task>(TasksLimit * 2); for (var j = 0; j < TasksLimit && i + j < keys.Count; j++) { var key1 = Path.Combine(KeyHelper.GetDatabaseSchema(), keys[i + j]); tasks.Add(RestoreFromDumpFile(dataReader, key1).ContinueWith(r => RestoreFromDumpFile(dataReader, KeyHelper.GetDatabaseData(key1.Substring(keyBase.Length + 1))))); } Task.WaitAll(tasks.ToArray()); } var comparer = new SqlComparer(); foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer)) { RunMysqlFile(u, true); SetStepCompleted(); } }
public void LoadFrom(IEnumerable<XElement> elements, int tenant, string[] configs, IDataReadOperator reader) { try { RestartService("TeamlabOfficeServer"); KillProcesses("w3wp"); KillProcesses("aspnet_wp"); } catch { } finally { OnProgressChanged("OK", 100); } }
public void LoadFrom(IEnumerable<XElement> elements, int tenant, string[] configs, IDataReadOperator reader) { var connectionKeys = new List<string>(); foreach (var connectionString in GetConnectionStrings(configs)) { var connectionKey = connectionString.ProviderName + connectionString.ConnectionString; if (connectionKeys.Contains(connectionKey)) continue; connectionKeys.Add(connectionKey); OnProgressChanged("Restoring database " + connectionString.Name, -1); RestoreDatabase(-1, connectionString, elements, reader); OnProgressChanged("OK", 100); } }
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory) { if (reader == null) throw new ArgumentNullException("reader"); if (columnMapper == null) throw new ArgumentNullException("columnMapper"); if (factory == null) throw new ArgumentNullException("factory"); _module = module; _reader = reader; _columnMapper = columnMapper; _factory = factory; }
private async Task RestoreFromDumpFile(IDataReadOperator dataReader, string fileName1, string fileName2 = null, string db = "default") { Logger.DebugFormat("Restore from {0}", fileName1); using (var stream = dataReader.GetEntry(fileName1)) { await RunMysqlFile(stream, db); } SetStepCompleted(); Logger.DebugFormat("Restore from {0}", fileName2); if (fileName2 != null) { using (var stream = dataReader.GetEntry(fileName2)) { await RunMysqlFile(stream, db); } SetStepCompleted(); } }
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory) { if (reader == null) { throw new ArgumentNullException("reader"); } if (columnMapper == null) { throw new ArgumentNullException("columnMapper"); } if (factory == null) { throw new ArgumentNullException("factory"); } _module = module; _reader = reader; _columnMapper = columnMapper; _factory = factory; }
private void DoRestoreStorage(IDataReadOperator dataReader) { InvokeInfo("begin restore storage"); var fileGroups = GetFilesToProcess(dataReader).GroupBy(file => file.Module).ToList(); int groupsProcessed = 0; foreach (var group in fileGroups) { IDataStore storage = StorageFactory.GetStorage(ConfigPath, _columnMapper.GetTenantMapping().ToString(), group.Key, null, null); foreach (BackupFileInfo file in group) { string adjustedPath = file.Path; IModuleSpecifics module = ModuleProvider.GetByStorageModule(file.Module); if (module == null || module.TryAdjustFilePath(_columnMapper, ref adjustedPath)) { Stream stream = dataReader.GetEntry(KeyHelper.GetFileZipKey(file)); try { storage.Save(file.Domain, adjustedPath, stream); } catch (Exception error) { InvokeWarning("can't restore file ({0}:{1}): {2}", file.Module, file.Path, error); } } } SetStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count)); } if (fileGroups.Count == 0) { SetStepCompleted(); } InvokeInfo("end restore storage"); }
private void RestoreFromDump(IDataReadOperator dataReader, DbFactory dbFactory) { var keyBase = KeyHelper.GetDatabaseSchema(); var keys = dataReader.Entries.Where(r => r.StartsWith(keyBase)).ToList(); var upgrades = new List <string>(); if (!string.IsNullOrEmpty(UpgradesPath) && Directory.Exists(UpgradesPath)) { upgrades = Directory.GetFiles(UpgradesPath).ToList(); } var stepscount = keys.Count * 2 + upgrades.Count; SetStepsCount(ProcessStorage ? stepscount + 1 : stepscount); for (var i = 0; i < keys.Count; i += TasksLimit) { var tasks = new List <Task>(TasksLimit * 2); for (var j = 0; j < TasksLimit && i + j < keys.Count; j++) { var key1 = keys[i + j]; tasks.Add(RestoreFromDumpFile(dataReader, key1).ContinueWith(r => RestoreFromDumpFile(dataReader, KeyHelper.GetDatabaseData(key1.Substring(keyBase.Length + 1))))); } Task.WaitAll(tasks.ToArray()); } var comparer = new SqlComparer(); foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer)) { RunMysqlFile(dbFactory, u, true); SetStepCompleted(); } }
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator reader) { processedTables.Clear(); foreach (var connectionString in GetConnectionStrings(configs)) { RestoreDatabase(connectionString, elements, reader); } }
private void RestoreDatabase(ConnectionStringSettings connectionString, IEnumerable <XElement> elements, IDataReadOperator reader) { var dbName = connectionString.Name; var dbElement = elements.SingleOrDefault(e => string.Compare(e.Name.LocalName, connectionString.Name, true) == 0); if (dbElement != null && dbElement.Attribute("ref") != null) { dbName = dbElement.Attribute("ref").Value; dbElement = elements.Single(e => string.Compare(e.Name.LocalName, dbElement.Attribute("ref").Value, true) == 0); } if (dbElement == null) { return; } var tables = dbHelper.GetTables(); for (var i = 0; i < tables.Count; i++) { var table = tables[i]; OnProgressChanged(table, (int)(i / (double)tables.Count * 100)); if (processedTables.Contains(table, StringComparer.InvariantCultureIgnoreCase)) { continue; } if (dbElement.Element(table) != null) { using (var stream = reader.GetEntry(string.Format("{0}\\{1}\\{2}", Name, dbName, table).ToLower())) { var data = new DataTable(); data.ReadXml(stream); dbHelper.SetTable(data); } processedTables.Add(table); } } }
private void RestoreFromDump(IDataReadOperator dataReader) { var keyBase = KeyHelper.GetDatabaseSchema(); var keys = dataReader.Entries.Where(r => r.StartsWith(keyBase)).ToList(); var upgrades = new List <string>(); var upgradesPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), UpgradesPath)); if (!string.IsNullOrEmpty(upgradesPath) && Directory.Exists(upgradesPath)) { upgrades = Directory.GetFiles(upgradesPath).ToList(); } var stepscount = keys.Count * 2 + upgrades.Count; if (ProcessStorage) { var storageModules = StorageFactory.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed); var tenants = CoreContext.TenantManager.GetTenants(false); stepscount += storageModules.Count() * tenants.Count; SetStepsCount(stepscount + 1); DoDeleteStorage(storageModules, tenants); } else { SetStepsCount(stepscount); } for (var i = 0; i < keys.Count; i += TasksLimit) { var tasks = new List <Task>(TasksLimit * 2); for (var j = 0; j < TasksLimit && i + j < keys.Count; j++) { var key1 = keys[i + j]; tasks.Add(RestoreFromDumpFile(dataReader, key1).ContinueWith(r => RestoreFromDumpFile(dataReader, KeyHelper.GetDatabaseData(key1.Substring(keyBase.Length + 1))))); } Task.WaitAll(tasks.ToArray()); } var comparer = new SqlComparer(); foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer)) { using (var s = File.OpenRead(u)) { if (u.Contains(".upgrade.")) { RunMysqlFile(s, null).Wait(); } else if (u.Contains(".data") || u.Contains(".upgrade")) { RunMysqlProcedure(s).Wait(); } else { RunMysqlFile(s).Wait(); } } SetStepCompleted(); } }
public RestoreDbModuleTask(IOptionsMonitor <ILog> options, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump, StorageFactory storageFactory, StorageFactoryConfig storageFactoryConfig, ModuleProvider moduleProvider) : base(factory, options, storageFactory, storageFactoryConfig, moduleProvider) { Reader = reader ?? throw new ArgumentNullException("reader"); ColumnMapper = columnMapper ?? throw new ArgumentNullException("columnMapper"); DbFactory = factory ?? throw new ArgumentNullException("factory"); Module = module; ReplaceDate = replaceDate; Dump = dump; Init(-1, null); }
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator dataOperator) { InvokeProgressChanged("Restoring files started", -1); var config = GetWebConfig(configs); AppDomain.CurrentDomain.SetData(Constants.CustomDataDirectory, Path.GetDirectoryName(config)); var files = elements.Where(e => e.Name == "file"); double current = 0; foreach (var file in files) { var backupInfo = new FileBackupInfo(file); var entry = dataOperator.GetEntry(GetBackupPath(backupInfo)); var storage = StorageFactory.GetStorage(config, tenant.ToString(), backupInfo.Module, null, null); storage.Save(backupInfo.Domain, backupInfo.Path, entry); InvokeProgressChanged("Restoring files", current / files.Count() * 100); current++; } InvokeProgressChanged("OK", 100); }
public void LoadFrom(IEnumerable<XElement> elements, int tenant, string[] configs, IDataReadOperator dataOperator) { InvokeProgressChanged("Restoring files...", 0); var config = GetWebConfig(configs); AppDomain.CurrentDomain.SetData(Constants.CustomDataDirectory, Path.GetDirectoryName(config)); var files = elements.Where(e => e.Name == "file"); double current = 0; foreach (var file in files) { var backupInfo = new FileBackupInfo(file); if (allowedModules.Contains(backupInfo.Module)) { var entry = dataOperator.GetEntry(GetBackupPath(backupInfo)); var storage = StorageFactory.GetStorage(config, tenant.ToString(), backupInfo.Module, null, null); try { storage.Save(backupInfo.Domain, backupInfo.Path, entry); } catch (Exception error) { log.ErrorFormat("Can not restore file {0}: {1}", file, error); } InvokeProgressChanged("Restoring file " + backupInfo.Path, current++ / files.Count() * 100); } } }
private void RestoreDatabase(int tenant, ConnectionStringSettings connectionString, IEnumerable<XElement> elements, IDataReadOperator reader) { var dbName = connectionString.Name; var dbElement = elements.SingleOrDefault(e => string.Compare(e.Name.LocalName, connectionString.Name, true) == 0); if (dbElement != null && dbElement.Attribute("ref") != null) { dbName = dbElement.Attribute("ref").Value; dbElement = elements.Single(e => string.Compare(e.Name.LocalName, dbElement.Attribute("ref").Value, true) == 0); } if (dbElement == null) return; using (var dbHelper = new DbHelper(tenant, connectionString)) { var tables = dbHelper.GetTables(); for (int i = 0; i < tables.Count; i++) { var table = tables[i]; OnProgressChanged(table, (int)(i / (double)tables.Count * 100)); if (dbElement.Element(table) == null) continue; var stream = reader.GetEntry(string.Format("{0}\\{1}\\{2}", Name, dbName, table).ToLower()); var data = new DataTable(); data.ReadXml(stream); dbHelper.SetTable(data); } } }
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator dataOperator) { InvokeProgressChanged("Restoring files...", 0); var config = GetWebConfig(configs); var files = elements.Where(e => e.Name == "file"); double current = 0; foreach (var file in files) { var backupInfo = new FileBackupInfo(file); if (allowedModules.Contains(backupInfo.Module)) { using (var entry = dataOperator.GetEntry(GetBackupPath(backupInfo))) { var storage = StorageFactory.GetStorage(config, tenant.ToString(), backupInfo.Module, null); try { storage.Save(backupInfo.Domain, backupInfo.Path, entry); } catch (Exception error) { log.ErrorFormat("Can not restore file {0}: {1}", file, error); } } InvokeProgressChanged("Restoring file " + backupInfo.Path, current++ / files.Count() * 100); } } }
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator reader) { var connectionKeys = new List <string>(); foreach (var connectionString in GetConnectionStrings(configs)) { var connectionKey = connectionString.ProviderName + connectionString.ConnectionString; if (connectionKeys.Contains(connectionKey)) { continue; } connectionKeys.Add(connectionKey); OnProgressChanged("Restoring database " + connectionString.Name, -1); RestoreDatabase(-1, connectionString, elements, reader); OnProgressChanged("OK", 100); } }
private IEnumerable<BackupFileInfo> GetFilesToProcess(IDataReadOperator dataReader) { using (Stream stream = dataReader.GetEntry(KeyHelper.GetStorageRestoreInfoZipKey())) { if (stream == null) return Enumerable.Empty<BackupFileInfo>(); var restoreInfo = XElement.Load(new StreamReader(stream)); return restoreInfo.Elements("file").Select(BackupFileInfo.FromXElement).ToList(); } }
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator reader) { try { RestartService("OnlyOffice"); KillProcesses("w3wp"); KillProcesses("aspnet_wp"); } catch { } finally { OnProgressChanged("OK", 100); } }
private void RestoreFromDump(IDataReadOperator dataReader) { var keyBase = KeyHelper.GetDatabaseSchema(); var keys = dataReader.GetEntries(keyBase).Select(r => Path.GetFileName(r)).ToList(); var dbs = dataReader.GetDirectories("").Where(r => Path.GetFileName(r).StartsWith("mailservice")).Select(r => Path.GetFileName(r)).ToList(); var upgrades = new List <string>(); var upgradesPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), UpgradesPath)); if (!string.IsNullOrEmpty(upgradesPath) && Directory.Exists(upgradesPath)) { upgrades = Directory.GetFiles(upgradesPath).ToList(); } var stepscount = keys.Count * 2 + upgrades.Count; Dictionary <string, List <string> > databases = new Dictionary <string, List <string> >(); foreach (var db in dbs) { var keys1 = dataReader.GetEntries(db + "/" + keyBase).Select(k => Path.GetFileName(k)).ToList(); stepscount += keys1.Count() * 2; databases.Add(db, keys1); } if (ProcessStorage) { var storageModules = StorageFactory.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed); var tenants = CoreContext.TenantManager.GetTenants(false); stepscount += storageModules.Count() * tenants.Count; SetStepsCount(stepscount + 1); DoDeleteStorage(storageModules, tenants); } else { SetStepsCount(stepscount); } for (var i = 0; i < keys.Count; i += TasksLimit) { var tasks = new List <Task>(TasksLimit * 2); for (var j = 0; j < TasksLimit && i + j < keys.Count; j++) { var key1 = Path.Combine(KeyHelper.GetDatabaseSchema(), keys[i + j]); var key2 = Path.Combine(KeyHelper.GetDatabaseData(), keys[i + j]); tasks.Add(RestoreFromDumpFile(dataReader, key1, key2)); } Task.WaitAll(tasks.ToArray()); } using (var dbManager = DbManager.FromHttpContext("default", 100000)) { dbManager.ExecuteList("select id, connection_string from mail_server_server").ForEach(r => { RegisterDatabase((int)r[0], JsonConvert.DeserializeObject <Dictionary <string, object> >(Convert.ToString(r[1]))["DbConnection"].ToString()); }); } foreach (var database in databases) { for (var i = 0; i < database.Value.Count; i += TasksLimit) { var tasks = new List <Task>(TasksLimit * 2); for (var j = 0; j < TasksLimit && i + j < database.Value.Count; j++) { var key1 = Path.Combine(database.Key, KeyHelper.GetDatabaseSchema(), database.Value[i + j]); var key2 = Path.Combine(database.Key, KeyHelper.GetDatabaseData(), database.Value[i + j]); tasks.Add(RestoreFromDumpFile(dataReader, key1, key2, database.Key)); } Task.WaitAll(tasks.ToArray()); } } var comparer = new SqlComparer(); foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer)) { using (var s = File.OpenRead(u)) { if (u.Contains(".upgrade.")) { RunMysqlFile(s, null).Wait(); } else if (u.Contains(".data") || u.Contains(".upgrade")) { RunMysqlProcedure(s).Wait(); } else { RunMysqlFile(s, "default").Wait(); } } SetStepCompleted(); } }