private void RestoreFromDump(IDataReadOperator dataReader)
{
var keyBase = KeyHelper.GetDatabaseSchema();
var keys = dataReader.GetEntries(keyBase).Select(r => Path.GetFileName(r)).ToList();
var upgrades = new List <string>();
if (!string.IsNullOrEmpty(UpgradesPath) && Directory.Exists(UpgradesPath))
{
upgrades = Directory.GetFiles(UpgradesPath).ToList();
}
var stepscount = keys.Count * 2 + upgrades.Count;
SetStepsCount(ProcessStorage ? stepscount + 1 : stepscount);
if (ProcessStorage)
{
var storageModules = StorageFactoryConfig.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed);
var tenants = TenantManager.GetTenants(false);
stepscount += storageModules.Count() * tenants.Count;
SetStepsCount(stepscount + 1);
DoDeleteStorage(storageModules, tenants);
}
else
{
SetStepsCount(stepscount);
}
for (var i = 0; i < keys.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < keys.Count; j++)
{
var key1 = Path.Combine(KeyHelper.GetDatabaseSchema(), keys[i + j]);
tasks.Add(RestoreFromDumpFile(dataReader, key1).ContinueWith(r => RestoreFromDumpFile(dataReader, KeyHelper.GetDatabaseData(key1.Substring(keyBase.Length + 1)))));
}
Task.WaitAll(tasks.ToArray());
}
var comparer = new SqlComparer();
foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer))
{
RunMysqlFile(u, true);
SetStepCompleted();
}
}
private void RestoreFromDump(IDataReadOperator dataReader, DbFactory dbFactory)
{
var keyBase = KeyHelper.GetDatabaseSchema();
var keys = dataReader.Entries.Where(r => r.StartsWith(keyBase)).ToList();
var upgrades = new List <string>();
if (!string.IsNullOrEmpty(UpgradesPath) && Directory.Exists(UpgradesPath))
{
upgrades = Directory.GetFiles(UpgradesPath).ToList();
}
var stepscount = keys.Count * 2 + upgrades.Count;
SetStepsCount(ProcessStorage ? stepscount + 1 : stepscount);
for (var i = 0; i < keys.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < keys.Count; j++)
{
var key1 = keys[i + j];
tasks.Add(RestoreFromDumpFile(dataReader, key1).ContinueWith(r => RestoreFromDumpFile(dataReader, KeyHelper.GetDatabaseData(key1.Substring(keyBase.Length + 1)))));
}
Task.WaitAll(tasks.ToArray());
}
var comparer = new SqlComparer();
foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer))
{
RunMysqlFile(dbFactory, u, true);
SetStepCompleted();
}
}
private void DoDump(IDataWriteOperator writer)
{
var tmp = Path.GetTempFileName();
File.AppendAllText(tmp, true.ToString());
writer.WriteEntry(KeyHelper.GetDumpKey(), tmp);
List <string> tables;
var files = new List <BackupFileInfo>();
using (var dbManager = new DbManager("default", 100000))
{
tables = dbManager.ExecuteList("show tables;").Select(r => Convert.ToString(r[0])).ToList();
}
var stepscount = tables.Count * 4; // (schema + data) * (dump + zip)
if (ProcessStorage)
{
var tenants = CoreContext.TenantManager.GetTenants(false).Select(r => r.TenantId);
foreach (var t in tenants)
{
files.AddRange(GetFiles(t));
}
stepscount += files.Count * 2 + 1;
Logger.Debug("files:" + files.Count);
}
SetStepsCount(stepscount);
var excluded = ModuleProvider.AllModules.Where(r => IgnoredModules.Contains(r.ModuleName)).SelectMany(r => r.Tables).Select(r => r.Name).ToList();
excluded.AddRange(IgnoredTables);
excluded.Add("res_");
var dir = Path.GetDirectoryName(BackupFilePath);
var subDir = Path.Combine(dir, Path.GetFileNameWithoutExtension(BackupFilePath));
var schemeDir = Path.Combine(subDir, KeyHelper.GetDatabaseSchema());
var dataDir = Path.Combine(subDir, KeyHelper.GetDatabaseData());
if (!Directory.Exists(schemeDir))
{
Directory.CreateDirectory(schemeDir);
}
if (!Directory.Exists(dataDir))
{
Directory.CreateDirectory(dataDir);
}
var dict = tables.ToDictionary(t => t, SelectCount);
tables.Sort((pair1, pair2) => dict[pair1].CompareTo(dict[pair2]));
for (var i = 0; i < tables.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < tables.Count; j++)
{
var t = tables[i + j];
tasks.Add(Task.Run(() => DumpTableScheme(t, schemeDir)));
if (!excluded.Any(t.StartsWith))
{
tasks.Add(Task.Run(() => DumpTableData(t, dataDir, dict[t])));
}
else
{
SetStepCompleted(2);
}
}
Task.WaitAll(tasks.ToArray());
ArchiveDir(writer, subDir);
}
Logger.DebugFormat("dir remove start {0}", subDir);
Directory.Delete(subDir, true);
Logger.DebugFormat("dir remove end {0}", subDir);
if (ProcessStorage)
{
DoDumpStorage(writer, files);
}
}
private void RestoreFromDump(IDataReadOperator dataReader)
{
var keyBase = KeyHelper.GetDatabaseSchema();
var keys = dataReader.Entries.Where(r => r.StartsWith(keyBase)).ToList();
var upgrades = new List <string>();
var upgradesPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), UpgradesPath));
if (!string.IsNullOrEmpty(upgradesPath) && Directory.Exists(upgradesPath))
{
upgrades = Directory.GetFiles(upgradesPath).ToList();
}
var stepscount = keys.Count * 2 + upgrades.Count;
if (ProcessStorage)
{
var storageModules = StorageFactory.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed);
var tenants = CoreContext.TenantManager.GetTenants(false);
stepscount += storageModules.Count() * tenants.Count;
SetStepsCount(stepscount + 1);
DoDeleteStorage(storageModules, tenants);
}
else
{
SetStepsCount(stepscount);
}
for (var i = 0; i < keys.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < keys.Count; j++)
{
var key1 = keys[i + j];
tasks.Add(RestoreFromDumpFile(dataReader, key1).ContinueWith(r => RestoreFromDumpFile(dataReader, KeyHelper.GetDatabaseData(key1.Substring(keyBase.Length + 1)))));
}
Task.WaitAll(tasks.ToArray());
}
var comparer = new SqlComparer();
foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer))
{
using (var s = File.OpenRead(u))
{
if (u.Contains(".upgrade."))
{
RunMysqlFile(s, null).Wait();
}
else if (u.Contains(".data") || u.Contains(".upgrade"))
{
RunMysqlProcedure(s).Wait();
}
else
{
RunMysqlFile(s).Wait();
}
}
SetStepCompleted();
}
}
private void RestoreFromDump(IDataReadOperator dataReader)
{
var keyBase = KeyHelper.GetDatabaseSchema();
var keys = dataReader.GetEntries(keyBase).Select(r => Path.GetFileName(r)).ToList();
var dbs = dataReader.GetDirectories("").Where(r => Path.GetFileName(r).StartsWith("mailservice")).Select(r => Path.GetFileName(r)).ToList();
var upgrades = new List <string>();
var upgradesPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), UpgradesPath));
if (!string.IsNullOrEmpty(upgradesPath) && Directory.Exists(upgradesPath))
{
upgrades = Directory.GetFiles(upgradesPath).ToList();
}
var stepscount = keys.Count * 2 + upgrades.Count;
Dictionary <string, List <string> > databases = new Dictionary <string, List <string> >();
foreach (var db in dbs)
{
var keys1 = dataReader.GetEntries(db + "/" + keyBase).Select(k => Path.GetFileName(k)).ToList();
stepscount += keys1.Count() * 2;
databases.Add(db, keys1);
}
if (ProcessStorage)
{
var storageModules = StorageFactory.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed);
var tenants = CoreContext.TenantManager.GetTenants(false);
stepscount += storageModules.Count() * tenants.Count;
SetStepsCount(stepscount + 1);
DoDeleteStorage(storageModules, tenants);
}
else
{
SetStepsCount(stepscount);
}
for (var i = 0; i < keys.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < keys.Count; j++)
{
var key1 = Path.Combine(KeyHelper.GetDatabaseSchema(), keys[i + j]);
var key2 = Path.Combine(KeyHelper.GetDatabaseData(), keys[i + j]);
tasks.Add(RestoreFromDumpFile(dataReader, key1, key2));
}
Task.WaitAll(tasks.ToArray());
}
using (var dbManager = DbManager.FromHttpContext("default", 100000))
{
dbManager.ExecuteList("select id, connection_string from mail_server_server").ForEach(r =>
{
RegisterDatabase((int)r[0], JsonConvert.DeserializeObject <Dictionary <string, object> >(Convert.ToString(r[1]))["DbConnection"].ToString());
});
}
foreach (var database in databases)
{
for (var i = 0; i < database.Value.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < database.Value.Count; j++)
{
var key1 = Path.Combine(database.Key, KeyHelper.GetDatabaseSchema(), database.Value[i + j]);
var key2 = Path.Combine(database.Key, KeyHelper.GetDatabaseData(), database.Value[i + j]);
tasks.Add(RestoreFromDumpFile(dataReader, key1, key2, database.Key));
}
Task.WaitAll(tasks.ToArray());
}
}
var comparer = new SqlComparer();
foreach (var u in upgrades.OrderBy(Path.GetFileName, comparer))
{
using (var s = File.OpenRead(u))
{
if (u.Contains(".upgrade."))
{
RunMysqlFile(s, null).Wait();
}
else if (u.Contains(".data") || u.Contains(".upgrade"))
{
RunMysqlProcedure(s).Wait();
}
else
{
RunMysqlFile(s, "default").Wait();
}
}
SetStepCompleted();
}
}
private void DoDump(IDataWriteOperator writer, string dbName, List <string> tables)
{
var excluded = ModuleProvider.AllModules.Where(r => IgnoredModules.Contains(r.ModuleName)).SelectMany(r => r.Tables).Select(r => r.Name).ToList();
excluded.AddRange(IgnoredTables);
excluded.Add("res_");
var dir = Path.GetDirectoryName(BackupFilePath);
var subDir = Path.Combine(dir, Path.GetFileNameWithoutExtension(BackupFilePath));
var schemeDir = "";
var dataDir = "";
if (dbName == "default")
{
schemeDir = Path.Combine(subDir, KeyHelper.GetDatabaseSchema());
dataDir = Path.Combine(subDir, KeyHelper.GetDatabaseData());
}
else
{
schemeDir = Path.Combine(subDir, dbName, KeyHelper.GetDatabaseSchema());
dataDir = Path.Combine(subDir, dbName, KeyHelper.GetDatabaseData());
}
if (!Directory.Exists(schemeDir))
{
Directory.CreateDirectory(schemeDir);
}
if (!Directory.Exists(dataDir))
{
Directory.CreateDirectory(dataDir);
}
var dict = new Dictionary <string, int>();
foreach (var table in tables)
{
dict.Add(table, SelectCount(table, dbName));
}
tables.Sort((pair1, pair2) => dict[pair1].CompareTo(dict[pair2]));
for (var i = 0; i < tables.Count; i += TasksLimit)
{
var tasks = new List <Task>(TasksLimit * 2);
for (var j = 0; j < TasksLimit && i + j < tables.Count; j++)
{
var t = tables[i + j];
tasks.Add(Task.Run(() => DumpTableScheme(t, schemeDir, dbName)));
if (!excluded.Any(t.StartsWith))
{
tasks.Add(Task.Run(() => DumpTableData(t, dataDir, dict[t], dbName, writer)));
}
else
{
SetStepCompleted(2);
}
}
Task.WaitAll(tasks.ToArray());
ArchiveDir(writer, subDir);
}
}