public RestoreDbModuleTask(ILog logger, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump)
: base(logger, -1, null)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (columnMapper == null)
{
throw new ArgumentNullException("columnMapper");
}
if (factory == null)
{
throw new ArgumentNullException("factory");
}
_module = module;
_reader = reader;
_columnMapper = columnMapper;
_factory = factory;
_replaceDate = replaceDate;
this.dump = dump;
}
private void DoRestoreStorage(IDataReadOperator dataReader)
{
InvokeInfo("begin restore storage");
var fileGroups = GetFilesToProcess(dataReader).GroupBy(file => file.Module).ToList();
int groupsProcessed = 0;
foreach (var group in fileGroups)
{
IDataStore storage = StorageFactory.GetStorage(ConfigPath, _columnMapper.GetTenantMapping().ToString(), group.Key, null, null);
foreach (BackupFileInfo file in group)
{
string adjustedPath = file.Path;
IModuleSpecifics module = ModuleProvider.GetByStorageModule(file.Module);
if (module == null || module.TryAdjustFilePath(_columnMapper, ref adjustedPath))
{
Stream stream = dataReader.GetEntry(KeyHelper.GetFileZipKey(file));
try
{
storage.Save(file.Domain, adjustedPath, stream);
}
catch (Exception error)
{
InvokeWarning("can't restore file ({0}:{1}): {2}", file.Module, file.Path, error);
}
}
}
SetStepProgress((int)(++groupsProcessed*100/(double)fileGroups.Count));
}
if (fileGroups.Count == 0)
SetStepCompleted();
InvokeInfo("end restore storage");
}
public void LoadFrom(IEnumerable<XElement> elements, int tenant, string[] configs, IDataReadOperator reader)
{
processedTables.Clear();
foreach (var connectionString in GetConnectionStrings(configs))
{
RestoreDatabase(connectionString, elements, reader);
}
}
private async Task RestoreFromDumpFile(IDataReadOperator dataReader, string fileName)
{
Logger.DebugFormat("Restore from {0}", fileName);
using (var stream = dataReader.GetEntry(fileName))
{
await RunMysqlFile(stream);
}
SetStepCompleted();
}
private void DoRestoreStorage(IDataReadOperator dataReader)
{
Logger.Debug("begin restore storage");
var fileGroups = GetFilesToProcess(dataReader).GroupBy(file => file.Module).ToList();
var groupsProcessed = 0;
foreach (var group in fileGroups)
{
foreach (var file in group)
{
var storage = StorageFactory.GetStorage(ConfigPath, Dump ? file.Tenant.ToString() : _columnMapper.GetTenantMapping().ToString(), group.Key);
var quotaController = storage.QuotaController;
storage.SetQuotaController(null);
try
{
var adjustedPath = file.Path;
var module = ModuleProvider.GetByStorageModule(file.Module, file.Domain);
if (module == null || module.TryAdjustFilePath(Dump, _columnMapper, ref adjustedPath))
{
var key = file.GetZipKey();
if (Dump)
{
key = Path.Combine(KeyHelper.GetStorage(), key);
}
using (var stream = dataReader.GetEntry(key))
{
try
{
storage.Save(file.Domain, adjustedPath, module != null ? module.PrepareData(key, stream, _columnMapper) : stream);
}
catch (Exception error)
{
Logger.WarnFormat("can't restore file ({0}:{1}): {2}", file.Module, file.Path, error);
}
}
}
}
finally
{
if (quotaController != null)
{
storage.SetQuotaController(quotaController);
}
}
}
SetCurrentStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
}
if (fileGroups.Count == 0)
{
SetStepCompleted();
}
Logger.Debug("end restore storage");
}
private IEnumerable <BackupFileInfo> GetFilesToProcess(IDataReadOperator dataReader)
{
using var stream = dataReader.GetEntry(KeyHelper.GetStorageRestoreInfoZipKey());
if (stream == null)
{
return(Enumerable.Empty <BackupFileInfo>());
}
var restoreInfo = XElement.Load(new StreamReader(stream));
return(restoreInfo.Elements("file").Select(BackupFileInfo.FromXElement).ToList());
}
private void RestoreFromDump(IDataReadOperator dataReader)
{
var keyBase = KeyHelper.GetDatabaseSchema();
var keys = dataReader.GetEntries(keyBase).Select(r => Path.GetFileName(r)).ToList();
var upgrades = new List <string>();
if (!string.IsNullOrEmpty(UpgradesPath) && Directory.Exists(UpgradesPath))
{
upgrades = Directory.GetFiles(UpgradesPath).ToList();
}
var stepscount = keys.Count * 2 + upgrades.Count;
SetStepsCount(ProcessStorage ? stepscount + 1 : stepscount);
if (ProcessStorage)
{
var storageModules = StorageFactoryConfig.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed);
var tenants = TenantManager.GetTenants(false);
stepscount += storageModules.Count() * tenants.Count;
SetStepsCount(stepscount + 1);
DoDeleteStorage(storageModules, tenants);
}
else
{
SetStepsCount(stepscount);
}
for (var i = 0; i < keys.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < keys.Count; j++)
{
var key1 = Path.Combine(KeyHelper.GetDatabaseSchema(), keys[i + j]);
tasks.Add(RestoreFromDumpFile(dataReader, key1).ContinueWith(r => RestoreFromDumpFile(dataReader, KeyHelper.GetDatabaseData(key1.Substring(keyBase.Length + 1)))));
}
Task.WaitAll(tasks.ToArray());
}
var comparer = new SqlComparer();
foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer))
{
RunMysqlFile(u, true);
SetStepCompleted();
}
}
public void LoadFrom(IEnumerable<XElement> elements, int tenant, string[] configs, IDataReadOperator reader)
{
try
{
RestartService("TeamlabOfficeServer");
KillProcesses("w3wp");
KillProcesses("aspnet_wp");
}
catch { }
finally
{
OnProgressChanged("OK", 100);
}
}
public void LoadFrom(IEnumerable<XElement> elements, int tenant, string[] configs, IDataReadOperator reader)
{
var connectionKeys = new List<string>();
foreach (var connectionString in GetConnectionStrings(configs))
{
var connectionKey = connectionString.ProviderName + connectionString.ConnectionString;
if (connectionKeys.Contains(connectionKey)) continue;
connectionKeys.Add(connectionKey);
OnProgressChanged("Restoring database " + connectionString.Name, -1);
RestoreDatabase(-1, connectionString, elements, reader);
OnProgressChanged("OK", 100);
}
}
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory)
{
if (reader == null)
throw new ArgumentNullException("reader");
if (columnMapper == null)
throw new ArgumentNullException("columnMapper");
if (factory == null)
throw new ArgumentNullException("factory");
_module = module;
_reader = reader;
_columnMapper = columnMapper;
_factory = factory;
}
private async Task RestoreFromDumpFile(IDataReadOperator dataReader, string fileName1, string fileName2 = null, string db = "default")
{
Logger.DebugFormat("Restore from {0}", fileName1);
using (var stream = dataReader.GetEntry(fileName1))
{
await RunMysqlFile(stream, db);
}
SetStepCompleted();
Logger.DebugFormat("Restore from {0}", fileName2);
if (fileName2 != null)
{
using (var stream = dataReader.GetEntry(fileName2))
{
await RunMysqlFile(stream, db);
}
SetStepCompleted();
}
}
public RestoreDbModuleTask(IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (columnMapper == null)
{
throw new ArgumentNullException("columnMapper");
}
if (factory == null)
{
throw new ArgumentNullException("factory");
}
_module = module;
_reader = reader;
_columnMapper = columnMapper;
_factory = factory;
}
private void DoRestoreStorage(IDataReadOperator dataReader)
{
InvokeInfo("begin restore storage");
var fileGroups = GetFilesToProcess(dataReader).GroupBy(file => file.Module).ToList();
int groupsProcessed = 0;
foreach (var group in fileGroups)
{
IDataStore storage = StorageFactory.GetStorage(ConfigPath, _columnMapper.GetTenantMapping().ToString(), group.Key, null, null);
foreach (BackupFileInfo file in group)
{
string adjustedPath = file.Path;
IModuleSpecifics module = ModuleProvider.GetByStorageModule(file.Module);
if (module == null || module.TryAdjustFilePath(_columnMapper, ref adjustedPath))
{
Stream stream = dataReader.GetEntry(KeyHelper.GetFileZipKey(file));
try
{
storage.Save(file.Domain, adjustedPath, stream);
}
catch (Exception error)
{
InvokeWarning("can't restore file ({0}:{1}): {2}", file.Module, file.Path, error);
}
}
}
SetStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
}
if (fileGroups.Count == 0)
{
SetStepCompleted();
}
InvokeInfo("end restore storage");
}
private void RestoreFromDump(IDataReadOperator dataReader, DbFactory dbFactory)
{
var keyBase = KeyHelper.GetDatabaseSchema();
var keys = dataReader.Entries.Where(r => r.StartsWith(keyBase)).ToList();
var upgrades = new List <string>();
if (!string.IsNullOrEmpty(UpgradesPath) && Directory.Exists(UpgradesPath))
{
upgrades = Directory.GetFiles(UpgradesPath).ToList();
}
var stepscount = keys.Count * 2 + upgrades.Count;
SetStepsCount(ProcessStorage ? stepscount + 1 : stepscount);
for (var i = 0; i < keys.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < keys.Count; j++)
{
var key1 = keys[i + j];
tasks.Add(RestoreFromDumpFile(dataReader, key1).ContinueWith(r => RestoreFromDumpFile(dataReader, KeyHelper.GetDatabaseData(key1.Substring(keyBase.Length + 1)))));
}
Task.WaitAll(tasks.ToArray());
}
var comparer = new SqlComparer();
foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer))
{
RunMysqlFile(dbFactory, u, true);
SetStepCompleted();
}
}
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator reader)
{
processedTables.Clear();
foreach (var connectionString in GetConnectionStrings(configs))
{
RestoreDatabase(connectionString, elements, reader);
}
}
private void RestoreDatabase(ConnectionStringSettings connectionString, IEnumerable <XElement> elements, IDataReadOperator reader)
{
var dbName = connectionString.Name;
var dbElement = elements.SingleOrDefault(e => string.Compare(e.Name.LocalName, connectionString.Name, true) == 0);
if (dbElement != null && dbElement.Attribute("ref") != null)
{
dbName = dbElement.Attribute("ref").Value;
dbElement = elements.Single(e => string.Compare(e.Name.LocalName, dbElement.Attribute("ref").Value, true) == 0);
}
if (dbElement == null)
{
return;
}
var tables = dbHelper.GetTables();
for (var i = 0; i < tables.Count; i++)
{
var table = tables[i];
OnProgressChanged(table, (int)(i / (double)tables.Count * 100));
if (processedTables.Contains(table, StringComparer.InvariantCultureIgnoreCase))
{
continue;
}
if (dbElement.Element(table) != null)
{
using (var stream = reader.GetEntry(string.Format("{0}\\{1}\\{2}", Name, dbName, table).ToLower()))
{
var data = new DataTable();
data.ReadXml(stream);
dbHelper.SetTable(data);
}
processedTables.Add(table);
}
}
}
private void RestoreFromDump(IDataReadOperator dataReader)
{
var keyBase = KeyHelper.GetDatabaseSchema();
var keys = dataReader.Entries.Where(r => r.StartsWith(keyBase)).ToList();
var upgrades = new List <string>();
var upgradesPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), UpgradesPath));
if (!string.IsNullOrEmpty(upgradesPath) && Directory.Exists(upgradesPath))
{
upgrades = Directory.GetFiles(upgradesPath).ToList();
}
var stepscount = keys.Count * 2 + upgrades.Count;
if (ProcessStorage)
{
var storageModules = StorageFactory.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed);
var tenants = CoreContext.TenantManager.GetTenants(false);
stepscount += storageModules.Count() * tenants.Count;
SetStepsCount(stepscount + 1);
DoDeleteStorage(storageModules, tenants);
}
else
{
SetStepsCount(stepscount);
}
for (var i = 0; i < keys.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < keys.Count; j++)
{
var key1 = keys[i + j];
tasks.Add(RestoreFromDumpFile(dataReader, key1).ContinueWith(r => RestoreFromDumpFile(dataReader, KeyHelper.GetDatabaseData(key1.Substring(keyBase.Length + 1)))));
}
Task.WaitAll(tasks.ToArray());
}
var comparer = new SqlComparer();
foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer))
{
using (var s = File.OpenRead(u))
{
if (u.Contains(".upgrade."))
{
RunMysqlFile(s, null).Wait();
}
else if (u.Contains(".data") || u.Contains(".upgrade"))
{
RunMysqlProcedure(s).Wait();
}
else
{
RunMysqlFile(s).Wait();
}
}
SetStepCompleted();
}
}
public RestoreDbModuleTask(IOptionsMonitor <ILog> options, IModuleSpecifics module, IDataReadOperator reader, ColumnMapper columnMapper, DbFactory factory, bool replaceDate, bool dump, StorageFactory storageFactory, StorageFactoryConfig storageFactoryConfig, ModuleProvider moduleProvider)
: base(factory, options, storageFactory, storageFactoryConfig, moduleProvider)
{
Reader = reader ?? throw new ArgumentNullException("reader");
ColumnMapper = columnMapper ?? throw new ArgumentNullException("columnMapper");
DbFactory = factory ?? throw new ArgumentNullException("factory");
Module = module;
ReplaceDate = replaceDate;
Dump = dump;
Init(-1, null);
}
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator dataOperator)
{
InvokeProgressChanged("Restoring files started", -1);
var config = GetWebConfig(configs);
AppDomain.CurrentDomain.SetData(Constants.CustomDataDirectory, Path.GetDirectoryName(config));
var files = elements.Where(e => e.Name == "file");
double current = 0;
foreach (var file in files)
{
var backupInfo = new FileBackupInfo(file);
var entry = dataOperator.GetEntry(GetBackupPath(backupInfo));
var storage = StorageFactory.GetStorage(config, tenant.ToString(), backupInfo.Module, null, null);
storage.Save(backupInfo.Domain, backupInfo.Path, entry);
InvokeProgressChanged("Restoring files", current / files.Count() * 100);
current++;
}
InvokeProgressChanged("OK", 100);
}
public void LoadFrom(IEnumerable<XElement> elements, int tenant, string[] configs, IDataReadOperator dataOperator)
{
InvokeProgressChanged("Restoring files...", 0);
var config = GetWebConfig(configs);
AppDomain.CurrentDomain.SetData(Constants.CustomDataDirectory, Path.GetDirectoryName(config));
var files = elements.Where(e => e.Name == "file");
double current = 0;
foreach (var file in files)
{
var backupInfo = new FileBackupInfo(file);
if (allowedModules.Contains(backupInfo.Module))
{
var entry = dataOperator.GetEntry(GetBackupPath(backupInfo));
var storage = StorageFactory.GetStorage(config, tenant.ToString(), backupInfo.Module, null, null);
try
{
storage.Save(backupInfo.Domain, backupInfo.Path, entry);
}
catch (Exception error)
{
log.ErrorFormat("Can not restore file {0}: {1}", file, error);
}
InvokeProgressChanged("Restoring file " + backupInfo.Path, current++ / files.Count() * 100);
}
}
}
private void RestoreDatabase(int tenant, ConnectionStringSettings connectionString, IEnumerable<XElement> elements, IDataReadOperator reader)
{
var dbName = connectionString.Name;
var dbElement = elements.SingleOrDefault(e => string.Compare(e.Name.LocalName, connectionString.Name, true) == 0);
if (dbElement != null && dbElement.Attribute("ref") != null)
{
dbName = dbElement.Attribute("ref").Value;
dbElement = elements.Single(e => string.Compare(e.Name.LocalName, dbElement.Attribute("ref").Value, true) == 0);
}
if (dbElement == null) return;
using (var dbHelper = new DbHelper(tenant, connectionString))
{
var tables = dbHelper.GetTables();
for (int i = 0; i < tables.Count; i++)
{
var table = tables[i];
OnProgressChanged(table, (int)(i / (double)tables.Count * 100));
if (dbElement.Element(table) == null) continue;
var stream = reader.GetEntry(string.Format("{0}\\{1}\\{2}", Name, dbName, table).ToLower());
var data = new DataTable();
data.ReadXml(stream);
dbHelper.SetTable(data);
}
}
}
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator dataOperator)
{
InvokeProgressChanged("Restoring files...", 0);
var config = GetWebConfig(configs);
var files = elements.Where(e => e.Name == "file");
double current = 0;
foreach (var file in files)
{
var backupInfo = new FileBackupInfo(file);
if (allowedModules.Contains(backupInfo.Module))
{
using (var entry = dataOperator.GetEntry(GetBackupPath(backupInfo)))
{
var storage = StorageFactory.GetStorage(config, tenant.ToString(), backupInfo.Module, null);
try
{
storage.Save(backupInfo.Domain, backupInfo.Path, entry);
}
catch (Exception error)
{
log.ErrorFormat("Can not restore file {0}: {1}", file, error);
}
}
InvokeProgressChanged("Restoring file " + backupInfo.Path, current++ / files.Count() * 100);
}
}
}
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator reader)
{
var connectionKeys = new List <string>();
foreach (var connectionString in GetConnectionStrings(configs))
{
var connectionKey = connectionString.ProviderName + connectionString.ConnectionString;
if (connectionKeys.Contains(connectionKey))
{
continue;
}
connectionKeys.Add(connectionKey);
OnProgressChanged("Restoring database " + connectionString.Name, -1);
RestoreDatabase(-1, connectionString, elements, reader);
OnProgressChanged("OK", 100);
}
}
private IEnumerable<BackupFileInfo> GetFilesToProcess(IDataReadOperator dataReader)
{
using (Stream stream = dataReader.GetEntry(KeyHelper.GetStorageRestoreInfoZipKey()))
{
if (stream == null)
return Enumerable.Empty<BackupFileInfo>();
var restoreInfo = XElement.Load(new StreamReader(stream));
return restoreInfo.Elements("file").Select(BackupFileInfo.FromXElement).ToList();
}
}
public void LoadFrom(IEnumerable <XElement> elements, int tenant, string[] configs, IDataReadOperator reader)
{
try
{
RestartService("OnlyOffice");
KillProcesses("w3wp");
KillProcesses("aspnet_wp");
}
catch { }
finally
{
OnProgressChanged("OK", 100);
}
}
private void RestoreFromDump(IDataReadOperator dataReader)
{
var keyBase = KeyHelper.GetDatabaseSchema();
var keys = dataReader.GetEntries(keyBase).Select(r => Path.GetFileName(r)).ToList();
var dbs = dataReader.GetDirectories("").Where(r => Path.GetFileName(r).StartsWith("mailservice")).Select(r => Path.GetFileName(r)).ToList();
var upgrades = new List <string>();
var upgradesPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), UpgradesPath));
if (!string.IsNullOrEmpty(upgradesPath) && Directory.Exists(upgradesPath))
{
upgrades = Directory.GetFiles(upgradesPath).ToList();
}
var stepscount = keys.Count * 2 + upgrades.Count;
Dictionary <string, List <string> > databases = new Dictionary <string, List <string> >();
foreach (var db in dbs)
{
var keys1 = dataReader.GetEntries(db + "/" + keyBase).Select(k => Path.GetFileName(k)).ToList();
stepscount += keys1.Count() * 2;
databases.Add(db, keys1);
}
if (ProcessStorage)
{
var storageModules = StorageFactory.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed);
var tenants = CoreContext.TenantManager.GetTenants(false);
stepscount += storageModules.Count() * tenants.Count;
SetStepsCount(stepscount + 1);
DoDeleteStorage(storageModules, tenants);
}
else
{
SetStepsCount(stepscount);
}
for (var i = 0; i < keys.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < keys.Count; j++)
{
var key1 = Path.Combine(KeyHelper.GetDatabaseSchema(), keys[i + j]);
var key2 = Path.Combine(KeyHelper.GetDatabaseData(), keys[i + j]);
tasks.Add(RestoreFromDumpFile(dataReader, key1, key2));
}
Task.WaitAll(tasks.ToArray());
}
using (var dbManager = DbManager.FromHttpContext("default", 100000))
{
dbManager.ExecuteList("select id, connection_string from mail_server_server").ForEach(r =>
{
RegisterDatabase((int)r[0], JsonConvert.DeserializeObject <Dictionary <string, object> >(Convert.ToString(r[1]))["DbConnection"].ToString());
});
}
foreach (var database in databases)
{
for (var i = 0; i < database.Value.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < database.Value.Count; j++)
{
var key1 = Path.Combine(database.Key, KeyHelper.GetDatabaseSchema(), database.Value[i + j]);
var key2 = Path.Combine(database.Key, KeyHelper.GetDatabaseData(), database.Value[i + j]);
tasks.Add(RestoreFromDumpFile(dataReader, key1, key2, database.Key));
}
Task.WaitAll(tasks.ToArray());
}
}
var comparer = new SqlComparer();
foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer))
{
using (var s = File.OpenRead(u))
{
if (u.Contains(".upgrade."))
{
RunMysqlFile(s, null).Wait();
}
else if (u.Contains(".data") || u.Contains(".upgrade"))
{
RunMysqlProcedure(s).Wait();
}
else
{
RunMysqlFile(s, "default").Wait();
}
}
SetStepCompleted();
}
}
