private void DoBackupStorage(IDataWriteOperator writer, List <IGrouping <string, BackupFileInfo> > fileGroups)
{
Logger.Debug("begin backup storage");
foreach (var group in fileGroups)
{
var filesProcessed = 0;
var filesCount = group.Count();
var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), group.Key);
foreach (var file in group)
{
ActionInvoker.Try(state =>
{
var f = (BackupFileInfo)state;
using (var fileStream = storage.GetReadStream(f.Domain, f.Path))
{
var tmp = Path.GetTempFileName();
try
{
using (var tmpFile = File.OpenWrite(tmp))
{
fileStream.CopyTo(tmpFile);
}
writer.WriteEntry(KeyHelper.GetFileZipKey(file), tmp);
}
finally
{
if (File.Exists(tmp))
{
File.Delete(tmp);
}
}
}
}, file, 5, error => Logger.Warn("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));
SetCurrentStepProgress((int)(++filesProcessed * 100 / (double)filesCount));
}
}
var restoreInfoXml = new XElement(
"storage_restore",
fileGroups
.SelectMany(group => group.Select(file => (object)file.ToXElement()))
.ToArray());
var tmpPath = Path.GetTempFileName();
using (var tmpFile = File.OpenWrite(tmpPath))
{
restoreInfoXml.WriteTo(tmpFile);
}
writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath);
File.Delete(tmpPath);
Logger.Debug("end backup storage");
}
private IEnumerable <BackupFileInfo> GetFilesToProcess(IDataReadOperator dataReader)
{
using var stream = dataReader.GetEntry(KeyHelper.GetStorageRestoreInfoZipKey());
if (stream == null)
{
return(Enumerable.Empty <BackupFileInfo>());
}
var restoreInfo = XElement.Load(new StreamReader(stream));
return(restoreInfo.Elements("file").Select(BackupFileInfo.FromXElement).ToList());
}
private void DoBackupStorage(IDataWriteOperator writer)
{
InvokeInfo("begin backup storage", Tenant.TenantAlias);
var fileGroups = GetFilesToProcess().GroupBy(file => file.Module).ToList();
int groupsProcessed = 0;
foreach (var group in fileGroups)
{
IDataStore storage = StorageFactory.GetStorage(ConfigPath, Tenant.TenantId.ToString(), group.Key, null, null);
foreach (BackupFileInfo file in group)
{
Stream stream = writer.BeginWriteEntry(KeyHelper.GetFileZipKey(file));
int offset = 0;
ActionInvoker.Try(state =>
{
var f = (BackupFileInfo)state;
using (var fileStream = storage.GetReadStream(f.Domain, f.Path, offset))
{
var buffer = new byte[2048];
int readed;
while ((readed = fileStream.Read(buffer, 0, buffer.Length)) > 0)
{
stream.Write(buffer, 0, readed);
offset += readed;
}
}
}, file, 5, error => InvokeWarning("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));
writer.EndWriteEntry();
}
SetStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
}
if (fileGroups.Count == 0)
{
SetStepCompleted();
}
var restoreInfoXml = new XElement(
"storage_restore",
fileGroups
.SelectMany(group => group.Select(file => (object)file.ToXElement()))
.ToArray());
Stream restoreInfoStream = writer.BeginWriteEntry(KeyHelper.GetStorageRestoreInfoZipKey());
restoreInfoXml.WriteTo(restoreInfoStream);
writer.EndWriteEntry();
InvokeInfo("end backup storage", Tenant.TenantAlias);
}
private void DoDumpStorage(IDataWriteOperator writer, IReadOnlyList <BackupFileInfo> files)
{
Logger.Debug("begin backup storage");
var dir = Path.GetDirectoryName(BackupFilePath);
var subDir = Path.Combine(dir, Path.GetFileNameWithoutExtension(BackupFilePath));
for (var i = 0; i < files.Count; i += TasksLimit)
{
var storageDir = Path.Combine(subDir, KeyHelper.GetStorage());
if (!Directory.Exists(storageDir))
{
Directory.CreateDirectory(storageDir);
}
var tasks = new List <Task>(TasksLimit);
for (var j = 0; j < TasksLimit && i + j < files.Count; j++)
{
var t = files[i + j];
tasks.Add(Task.Run(() => DoDumpFile(t, storageDir)));
}
Task.WaitAll(tasks.ToArray());
ArchiveDir(writer, subDir);
Directory.Delete(storageDir, true);
}
var restoreInfoXml = new XElement("storage_restore", files.Select(file => (object)file.ToXElement()).ToArray());
var tmpPath = Path.Combine(subDir, KeyHelper.GetStorageRestoreInfoZipKey());
Directory.CreateDirectory(Path.GetDirectoryName(tmpPath));
using (var tmpFile = File.OpenWrite(tmpPath))
{
restoreInfoXml.WriteTo(tmpFile);
}
writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath);
File.Delete(tmpPath);
SetStepCompleted();
Directory.Delete(subDir, true);
Logger.Debug("end backup storage");
}
private void DoBackupStorage(IDataWriteOperator writer, DbFactory dbFactory)
{
Logger.Debug("begin backup storage");
var files = GetFilesToProcess();
var exclude = new List <string>();
using (var db = dbFactory.OpenConnection())
using (var command = db.CreateCommand())
{
command.CommandText = "select storage_path from backup_backup where tenant_id = " + TenantId + " and storage_type = 0 and storage_path is not null";
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
exclude.Add(reader.GetString(0));
}
}
}
files = files.Where(f => !exclude.Any(e => f.Path.Contains(string.Format("/file_{0}/", e))));
var fileGroups = files.GroupBy(file => file.Module).ToList();
var groupsProcessed = 0;
foreach (var group in fileGroups)
{
var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), group.Key);
foreach (var file in group)
{
ActionInvoker.Try(state =>
{
var f = (BackupFileInfo)state;
using (var fileStream = storage.GetReadStream(f.Domain, f.Path))
{
var tmp = Path.GetTempFileName();
try
{
using (var tmpFile = File.OpenWrite(tmp))
{
fileStream.CopyTo(tmpFile);
}
writer.WriteEntry(KeyHelper.GetFileZipKey(file), tmp);
}
finally
{
if (File.Exists(tmp))
{
File.Delete(tmp);
}
}
}
}, file, 5, error => Logger.Warn("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));
}
SetCurrentStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
}
if (fileGroups.Count == 0)
{
SetStepCompleted();
}
var restoreInfoXml = new XElement(
"storage_restore",
fileGroups
.SelectMany(group => group.Select(file => (object)file.ToXElement()))
.ToArray());
var tmpPath = Path.GetTempFileName();
using (var tmpFile = File.OpenWrite(tmpPath))
{
restoreInfoXml.WriteTo(tmpFile);
}
writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath);
File.Delete(tmpPath);
Logger.Debug("end backup storage");
}