private void DoBackupStorage(IDataWriteOperator writer, List <IGrouping <string, BackupFileInfo> > fileGroups) { Logger.Debug("begin backup storage"); foreach (var group in fileGroups) { var filesProcessed = 0; var filesCount = group.Count(); var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), group.Key); foreach (var file in group) { ActionInvoker.Try(state => { var f = (BackupFileInfo)state; using (var fileStream = storage.GetReadStream(f.Domain, f.Path)) { var tmp = Path.GetTempFileName(); try { using (var tmpFile = File.OpenWrite(tmp)) { fileStream.CopyTo(tmpFile); } writer.WriteEntry(KeyHelper.GetFileZipKey(file), tmp); } finally { if (File.Exists(tmp)) { File.Delete(tmp); } } } }, file, 5, error => Logger.Warn("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error)); SetCurrentStepProgress((int)(++filesProcessed * 100 / (double)filesCount)); } } var restoreInfoXml = new XElement( "storage_restore", fileGroups .SelectMany(group => group.Select(file => (object)file.ToXElement())) .ToArray()); var tmpPath = Path.GetTempFileName(); using (var tmpFile = File.OpenWrite(tmpPath)) { restoreInfoXml.WriteTo(tmpFile); } writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath); File.Delete(tmpPath); Logger.Debug("end backup storage"); }
public override void RunJob() { Logger.DebugFormat("begin restore data for module {0}", _module.ModuleName); SetStepsCount(_module.Tables.Count(t => !IgnoredTables.Contains(t.Name))); using (var connection = _factory.OpenConnection()) { foreach (var table in _module.GetTablesOrdered().Where(t => !IgnoredTables.Contains(t.Name) && t.InsertMethod != InsertMethod.None)) { Logger.DebugFormat("begin restore table {0}", table.Name); var transactionsCommited = 0; var rowsInserted = 0; ActionInvoker.Try( state => RestoreTable(connection.Fix(), (TableInfo)state, ref transactionsCommited, ref rowsInserted), table, 5, onAttemptFailure: error => _columnMapper.Rollback(), onFailure: error => { throw ThrowHelper.CantRestoreTable(table.Name, error); }); SetStepCompleted(); Logger.DebugFormat("{0} rows inserted for table {1}", rowsInserted, table.Name); } } Logger.DebugFormat("end restore data for module {0}", _module.ModuleName); }
private void DoDeleteStorage(IEnumerable <string> storageModules, IEnumerable <Tenant> tenants) { Logger.Debug("begin delete storage"); foreach (var tenant in tenants) { foreach (var module in storageModules) { var storage = StorageFactory.GetStorage(ConfigPath, tenant.TenantId.ToString(), module); var domains = StorageFactory.GetDomainList(ConfigPath, module).ToList(); domains.Add(string.Empty); //instead storage.DeleteFiles("\\", "*.*", true); foreach (var domain in domains) { ActionInvoker.Try( state => { if (storage.IsDirectory((string)state)) { storage.DeleteFiles((string)state, "\\", "*.*", true); } }, domain, 5, onFailure: error => Logger.WarnFormat("Can't delete files for domain {0}: \r\n{1}", domain, error) ); } SetStepCompleted(); } } Logger.Debug("end delete storage"); }
private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module) { Logger.Debug("begin saving data for module {0}", module.ModuleName); var tablesToProcess = module.Tables.Where(t => !IgnoredTables.Contains(t.Name) && t.InsertMethod != InsertMethod.None).ToList(); int tablesCount = tablesToProcess.Count; int tablesProcessed = 0; using (var connection = dbFactory.OpenConnection()) { foreach (var table in tablesToProcess) { Logger.Debug("begin load table {0}", table.Name); using (var data = new DataTable(table.Name)) { ActionInvoker.Try( state => { data.Clear(); var t = (TableInfo)state; var dataAdapter = dbFactory.CreateDataAdapter(); dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), TenantId, t).WithTimeout(600); ((DbDataAdapter)dataAdapter).Fill(data); }, table, maxAttempts: 5, onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); }, onAttemptFailure: error => Logger.Warn("backup attempt failure: {0}", error)); foreach (var col in data.Columns.Cast <DataColumn>().Where(col => col.DataType == typeof(DateTime))) { col.DateTimeMode = DataSetDateTime.Unspecified; } module.PrepareData(data); Logger.Debug("end load table {0}", table.Name); Logger.Debug("begin saving table {0}", table.Name); var tmp = Path.GetTempFileName(); using (var file = File.OpenWrite(tmp)) { data.WriteXml(file, XmlWriteMode.WriteSchema); data.Clear(); } writer.WriteEntry(KeyHelper.GetTableZipKey(module, data.TableName), tmp); File.Delete(tmp); Logger.Debug("end saving table {0}", table.Name); } SetCurrentStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount)); } } Logger.Debug("end saving data for module {0}", module.ModuleName); }
private void DoBackupStorage(IDataWriteOperator writer) { InvokeInfo("begin backup storage", Tenant.TenantAlias); var fileGroups = GetFilesToProcess().GroupBy(file => file.Module).ToList(); int groupsProcessed = 0; foreach (var group in fileGroups) { IDataStore storage = StorageFactory.GetStorage(ConfigPath, Tenant.TenantId.ToString(), group.Key, null, null); foreach (BackupFileInfo file in group) { Stream stream = writer.BeginWriteEntry(KeyHelper.GetFileZipKey(file)); int offset = 0; ActionInvoker.Try(state => { var f = (BackupFileInfo)state; using (var fileStream = storage.GetReadStream(f.Domain, f.Path, offset)) { var buffer = new byte[2048]; int readed; while ((readed = fileStream.Read(buffer, 0, buffer.Length)) > 0) { stream.Write(buffer, 0, readed); offset += readed; } } }, file, 5, error => InvokeWarning("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error)); writer.EndWriteEntry(); } SetStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count)); } if (fileGroups.Count == 0) { SetStepCompleted(); } var restoreInfoXml = new XElement( "storage_restore", fileGroups .SelectMany(group => group.Select(file => (object)file.ToXElement())) .ToArray()); Stream restoreInfoStream = writer.BeginWriteEntry(KeyHelper.GetStorageRestoreInfoZipKey()); restoreInfoXml.WriteTo(restoreInfoStream); writer.EndWriteEntry(); InvokeInfo("end backup storage", Tenant.TenantAlias); }
private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module) { InvokeInfo("begin saving data for module {0}", module.ModuleName); int tablesCount = module.Tables.Count(); int tablesProcessed = 0; using (var connection = dbFactory.OpenConnection(module.ConnectionStringName)) { foreach (var table in module.Tables) { InvokeInfo("begin saving table {0}", table.Name); using (var data = new DataTable(table.Name)) { ActionInvoker.Try( state => { data.Clear(); var t = (TableInfo)state; var dataAdapter = dbFactory.CreateDataAdapter(module.ConnectionStringName); dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(600); ((DbDataAdapter)dataAdapter).Fill(data); }, table, maxAttempts: 5, onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); }, onAttemptFailure: error => InvokeWarning("backup attempt failure: {0}", error)); foreach (var col in data.Columns.Cast <DataColumn>().Where(col => col.DataType == typeof(DateTime))) { col.DateTimeMode = DataSetDateTime.Unspecified; } var stream = writer.BeginWriteEntry(KeyHelper.GetTableZipKey(module, data.TableName)); data.WriteXml(stream, XmlWriteMode.WriteSchema); writer.EndWriteEntry(); data.Clear(); } SetStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount)); } } InvokeInfo("end saving data for module {0}", module.ModuleName); }
private void DoDeleteStorage() { InvokeInfo("begin delete storage"); List <string> storageModules = StorageFactory.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed).ToList(); int modulesProcessed = 0; foreach (string module in storageModules) { IDataStore storage = StorageFactory.GetStorage(ConfigPath, Tenant.TenantId.ToString(), module, null, null); List <string> domains = StorageFactory.GetDomainList(ConfigPath, module).ToList(); foreach (var domain in domains) { ActionInvoker.Try(state => storage.DeleteFiles((string)state, "\\", "*.*", true), domain, 5, onFailure: error => InvokeWarning("Can't delete files for domain {0}: \r\n{1}", domain, error)); } storage.DeleteFiles("\\", "*.*", true); SetStepProgress((int)((++modulesProcessed * 100) / (double)storageModules.Count)); } InvokeInfo("end delete storage"); }
private void DoDeleteModule(DbFactory dbFactory, IModuleSpecifics module) { InvokeInfo("begin delete data for module ({0})", module.ModuleName); int tablesCount = module.Tables.Count(); int tablesProcessed = 0; using (var connection = dbFactory.OpenConnection(module.ConnectionStringName)) { foreach (var table in module.GetTablesOrdered().Reverse().Where(t => !IgnoredTables.Contains(t.Name))) { ActionInvoker.Try(state => { var t = (TableInfo)state; module.CreateDeleteCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(120).ExecuteNonQuery(); }, table, 5, onFailure: error => { throw ThrowHelper.CantDeleteTable(table.Name, error); }); SetStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount)); } } InvokeInfo("end delete data for module ({0})", module.ModuleName); }
private void DoDeleteStorage() { Logger.Debug("begin delete storage"); var storageModules = StorageFactoryConfig.GetModuleList(ConfigPath).Where(IsStorageModuleAllowed).ToList(); var modulesProcessed = 0; foreach (var module in storageModules) { var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), module); var domains = StorageFactoryConfig.GetDomainList(ConfigPath, module).ToList(); foreach (var domain in domains) { ActionInvoker.Try(state => storage.DeleteFiles((string)state, "\\", "*.*", true), domain, 5, onFailure: error => Logger.WarnFormat("Can't delete files for domain {0}: \r\n{1}", domain, error)); } storage.DeleteFiles("\\", "*.*", true); SetCurrentStepProgress((int)((++modulesProcessed * 100) / (double)storageModules.Count)); } Logger.Debug("end delete storage"); }
private void DoBackupStorage(IDataWriteOperator writer, DbFactory dbFactory) { Logger.Debug("begin backup storage"); var files = GetFilesToProcess(); var exclude = new List <string>(); using (var db = dbFactory.OpenConnection()) using (var command = db.CreateCommand()) { command.CommandText = "select storage_path from backup_backup where tenant_id = " + TenantId + " and storage_type = 0 and storage_path is not null"; using (var reader = command.ExecuteReader()) { while (reader.Read()) { exclude.Add(reader.GetString(0)); } } } files = files.Where(f => !exclude.Any(e => f.Path.Contains(string.Format("/file_{0}/", e)))); var fileGroups = files.GroupBy(file => file.Module).ToList(); var groupsProcessed = 0; foreach (var group in fileGroups) { var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), group.Key); foreach (var file in group) { ActionInvoker.Try(state => { var f = (BackupFileInfo)state; using (var fileStream = storage.GetReadStream(f.Domain, f.Path)) { var tmp = Path.GetTempFileName(); try { using (var tmpFile = File.OpenWrite(tmp)) { fileStream.CopyTo(tmpFile); } writer.WriteEntry(KeyHelper.GetFileZipKey(file), tmp); } finally { if (File.Exists(tmp)) { File.Delete(tmp); } } } }, file, 5, error => Logger.Warn("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error)); } SetCurrentStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count)); } if (fileGroups.Count == 0) { SetStepCompleted(); } var restoreInfoXml = new XElement( "storage_restore", fileGroups .SelectMany(group => group.Select(file => (object)file.ToXElement())) .ToArray()); var tmpPath = Path.GetTempFileName(); using (var tmpFile = File.OpenWrite(tmpPath)) { restoreInfoXml.WriteTo(tmpFile); } writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath); File.Delete(tmpPath); Logger.Debug("end backup storage"); }