예제 #1
0
        public IEnumerable <XElement> GetElements(int tenant, string[] configs, IDataWriteOperator writer)
        {
            processedTables.Clear();
            var xml            = new List <XElement>();
            var connectionKeys = new Dictionary <string, string>();

            foreach (var connectionString in GetConnectionStrings(configs))
            {
                //do not save the base, having the same provider and connection string is not to duplicate
                //data, but also expose the ref attribute of repetitive bases for the correct recovery
                var node = new XElement(connectionString.Name);
                xml.Add(node);

                var connectionKey = connectionString.ProviderName + connectionString.ConnectionString;
                if (connectionKeys.ContainsKey(connectionKey))
                {
                    node.Add(new XAttribute("ref", connectionKeys[connectionKey]));
                }
                else
                {
                    connectionKeys.Add(connectionKey, connectionString.Name);
                    node.Add(BackupDatabase(tenant, connectionString, writer));
                }
            }

            return(xml);
        }
예제 #2
0
        private void DoBackupStorage(IDataWriteOperator writer, List <IGrouping <string, BackupFileInfo> > fileGroups)
        {
            Logger.Debug("begin backup storage");

            foreach (var group in fileGroups)
            {
                var filesProcessed = 0;
                var filesCount     = group.Count();

                var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), group.Key);
                foreach (var file in group)
                {
                    ActionInvoker.Try(state =>
                    {
                        var f = (BackupFileInfo)state;
                        using (var fileStream = storage.GetReadStream(f.Domain, f.Path))
                        {
                            var tmp = Path.GetTempFileName();
                            try
                            {
                                using (var tmpFile = File.OpenWrite(tmp))
                                {
                                    fileStream.CopyTo(tmpFile);
                                }

                                writer.WriteEntry(KeyHelper.GetFileZipKey(file), tmp);
                            }
                            finally
                            {
                                if (File.Exists(tmp))
                                {
                                    File.Delete(tmp);
                                }
                            }
                        }
                    }, file, 5, error => Logger.Warn("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));

                    SetCurrentStepProgress((int)(++filesProcessed * 100 / (double)filesCount));
                }
            }

            var restoreInfoXml = new XElement(
                "storage_restore",
                fileGroups
                .SelectMany(group => group.Select(file => (object)file.ToXElement()))
                .ToArray());

            var tmpPath = Path.GetTempFileName();

            using (var tmpFile = File.OpenWrite(tmpPath))
            {
                restoreInfoXml.WriteTo(tmpFile);
            }

            writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath);
            File.Delete(tmpPath);


            Logger.Debug("end backup storage");
        }
예제 #3
0
        public IEnumerable<XElement> GetElements(int tenant, string[] configs, IDataWriteOperator writer)
        {
            var xml = new List<XElement>();
            var connectionKeys = new Dictionary<string, string>();
            foreach (ConnectionStringSettings connectionString in GetConnectionStrings(configs))
            {
                //do not save the base, having the same provider and connection string is not to duplicate
                //data, but also expose the ref attribute of repetitive bases for the correct recovery
                var node = new XElement(connectionString.Name);
                xml.Add(node);

                var connectionKey = connectionString.ProviderName + connectionString.ConnectionString;
                if (connectionKeys.ContainsKey(connectionKey))
                {
                    node.Add(new XAttribute("ref", connectionKeys[connectionKey]));
                }
                else
                {
                    connectionKeys.Add(connectionKey, connectionString.Name);
                    OnProgressChanged("Saving database " + connectionString.Name, -1);
                    node.Add(BackupDatabase(tenant, connectionString, writer));
                    OnProgressChanged("OK", 100);
                }
            }

            return xml;
        }
예제 #4
0
        public IEnumerable <XElement> GetElements(int tenant, string[] configs, IDataWriteOperator writer)
        {
            InvokeProgressChanged("Saving files...", 0);

            var config = GetWebConfig(configs);
            var files  = ComposeFiles(tenant, config);

            var elements   = new List <XElement>();
            var backupKeys = new List <string>();

            var counter    = 0;
            var totalCount = (double)files.Count();

            foreach (var file in files)
            {
                var backupPath = GetBackupPath(file);
                if (!backupKeys.Contains(backupPath))
                {
                    var storage   = StorageFactory.GetStorage(config, tenant.ToString(), file.Module);
                    var zipStream = writer.BeginWriteEntry(backupPath);
                    var errors    = 0;
                    var offset    = 0;
                    while (true)
                    {
                        try
                        {
                            using (var stream = storage.GetReadStream(file.Domain, file.Path, offset))
                            {
                                var buffer = new byte[2048];
                                var readed = 0;
                                while (0 < (readed = stream.Read(buffer, 0, buffer.Length)))
                                {
                                    zipStream.Write(buffer, 0, readed);
                                    offset += readed;
                                }
                            }
                            break;
                        }
                        catch (Exception error)
                        {
                            errors++;
                            if (20 < errors)
                            {
                                log.ErrorFormat("Can not backup file {0}: {1}", file.Path, error);
                                break;
                            }
                        }
                    }
                    writer.EndWriteEntry();
                    elements.Add(file.ToXElement());
                    backupKeys.Add(backupPath);
                    log.DebugFormat("Backup file {0}", file.Path);
                }
                InvokeProgressChanged("Saving file " + file.Path, counter++ / totalCount * 100);
            }

            return(elements);
        }
        public IEnumerable<XElement> GetElements(int tenant, string[] configs, IDataWriteOperator writer)
        {
            InvokeProgressChanged("Saving files...", 0);

            var config = GetWebConfig(configs);
            var files = ComposeFiles(tenant, config);

            var elements = new List<XElement>();
            var backupKeys = new List<string>();

            var counter = 0;
            var totalCount = (double)files.Count();

            foreach (var file in files)
            {
                var backupPath = GetBackupPath(file);
                if (!backupKeys.Contains(backupPath))
                {
                    var storage = StorageFactory.GetStorage(config, tenant.ToString(), file.Module);
                    var zipStream = writer.BeginWriteEntry(backupPath);
                    var errors = 0;
                    var offset = 0;
                    while (true)
                    {
                        try
                        {
                            using (var stream = storage.GetReadStream(file.Domain, file.Path, offset))
                            {
                                var buffer = new byte[2048];
                                var readed = 0;
                                while (0 < (readed = stream.Read(buffer, 0, buffer.Length)))
                                {
                                    zipStream.Write(buffer, 0, readed);
                                    offset += readed;
                                }
                            }
                            break;
                        }
                        catch (Exception error)
                        {
                            errors++;
                            if (20 < errors)
                            {
                                log.ErrorFormat("Can not backup file {0}: {1}", file.Path, error);
                                break;
                            }
                        }
                    }
                    writer.EndWriteEntry();
                    elements.Add(file.ToXElement());
                    backupKeys.Add(backupPath);
                    log.DebugFormat("Backup file {0}", file.Path);
                }
                InvokeProgressChanged("Saving file " + file.Path, counter++ / totalCount * 100);
            }

            return elements;
        }
예제 #6
0
        public IEnumerable <XElement> GetElements(int tenant, string[] configs, IDataWriteOperator writer)
        {
            InvokeProgressChanged("Saving files...", 0);

            var config = GetWebConfig(configs);
            var files  = ComposeFiles(tenant, config);

            var elements   = new List <XElement>();
            var backupKeys = new List <string>();

            var counter    = 0;
            var totalCount = (double)files.Count();

            foreach (var file in files)
            {
                var backupPath = GetBackupPath(file);
                if (!backupKeys.Contains(backupPath))
                {
                    var storage = StorageFactory.GetStorage(config, tenant.ToString(), file.Module);
                    var errors  = 0;
                    while (true)
                    {
                        try
                        {
                            using (var stream = storage.GetReadStream(file.Domain, file.Path))
                            {
                                var tmpPath = Path.GetTempFileName();
                                using (var tmpFile = File.OpenWrite(tmpPath))
                                {
                                    stream.CopyTo(tmpFile);
                                }

                                writer.WriteEntry(backupPath, tmpPath);
                                File.Delete(tmpPath);
                            }

                            break;
                        }
                        catch (Exception error)
                        {
                            errors++;
                            if (20 < errors)
                            {
                                log.ErrorFormat("Can not backup file {0}: {1}", file.Path, error);
                                break;
                            }
                        }
                    }
                    elements.Add(file.ToXElement());
                    backupKeys.Add(backupPath);
                    log.DebugFormat("Backup file {0}", file.Path);
                }
                InvokeProgressChanged("Saving file " + file.Path, counter++ / totalCount * 100);
            }

            return(elements);
        }
예제 #7
0
        private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module)
        {
            Logger.Debug("begin saving data for module {0}", module.ModuleName);
            var tablesToProcess = module.Tables.Where(t => !IgnoredTables.Contains(t.Name) && t.InsertMethod != InsertMethod.None).ToList();
            int tablesCount     = tablesToProcess.Count;
            int tablesProcessed = 0;

            using (var connection = dbFactory.OpenConnection())
            {
                foreach (var table in tablesToProcess)
                {
                    Logger.Debug("begin load table {0}", table.Name);
                    using (var data = new DataTable(table.Name))
                    {
                        ActionInvoker.Try(
                            state =>
                        {
                            data.Clear();
                            var t                     = (TableInfo)state;
                            var dataAdapter           = dbFactory.CreateDataAdapter();
                            dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), TenantId, t).WithTimeout(600);
                            ((DbDataAdapter)dataAdapter).Fill(data);
                        },
                            table,
                            maxAttempts: 5,
                            onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); },
                            onAttemptFailure: error => Logger.Warn("backup attempt failure: {0}", error));

                        foreach (var col in data.Columns.Cast <DataColumn>().Where(col => col.DataType == typeof(DateTime)))
                        {
                            col.DateTimeMode = DataSetDateTime.Unspecified;
                        }

                        module.PrepareData(data);

                        Logger.Debug("end load table {0}", table.Name);

                        Logger.Debug("begin saving table {0}", table.Name);

                        var tmp = Path.GetTempFileName();
                        using (var file = File.OpenWrite(tmp))
                        {
                            data.WriteXml(file, XmlWriteMode.WriteSchema);
                            data.Clear();
                        }

                        writer.WriteEntry(KeyHelper.GetTableZipKey(module, data.TableName), tmp);
                        File.Delete(tmp);

                        Logger.Debug("end saving table {0}", table.Name);
                    }

                    SetCurrentStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount));
                }
            }
            Logger.Debug("end saving data for module {0}", module.ModuleName);
        }
예제 #8
0
 private void ArchiveDir(IDataWriteOperator writer, string subDir)
 {
     Logger.DebugFormat("archive dir start {0}", subDir);
     foreach (var enumerateFile in Directory.EnumerateFiles(subDir, "*", SearchOption.AllDirectories))
     {
         writer.WriteEntry(enumerateFile.Substring(subDir.Length), enumerateFile);
         File.Delete(enumerateFile);
         SetStepCompleted();
     }
     Logger.DebugFormat("archive dir end {0}", subDir);
 }
예제 #9
0
        private void DoBackupStorage(IDataWriteOperator writer)
        {
            InvokeInfo("begin backup storage", Tenant.TenantAlias);
            var fileGroups      = GetFilesToProcess().GroupBy(file => file.Module).ToList();
            int groupsProcessed = 0;

            foreach (var group in fileGroups)
            {
                IDataStore storage = StorageFactory.GetStorage(ConfigPath, Tenant.TenantId.ToString(), group.Key, null, null);
                foreach (BackupFileInfo file in group)
                {
                    Stream stream = writer.BeginWriteEntry(KeyHelper.GetFileZipKey(file));

                    int offset = 0;
                    ActionInvoker.Try(state =>
                    {
                        var f = (BackupFileInfo)state;
                        using (var fileStream = storage.GetReadStream(f.Domain, f.Path, offset))
                        {
                            var buffer = new byte[2048];
                            int readed;
                            while ((readed = fileStream.Read(buffer, 0, buffer.Length)) > 0)
                            {
                                stream.Write(buffer, 0, readed);
                                offset += readed;
                            }
                        }
                    }, file, 5, error => InvokeWarning("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));

                    writer.EndWriteEntry();
                }
                SetStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
            }

            if (fileGroups.Count == 0)
            {
                SetStepCompleted();
            }

            var restoreInfoXml = new XElement(
                "storage_restore",
                fileGroups
                .SelectMany(group => group.Select(file => (object)file.ToXElement()))
                .ToArray());

            Stream restoreInfoStream = writer.BeginWriteEntry(KeyHelper.GetStorageRestoreInfoZipKey());

            restoreInfoXml.WriteTo(restoreInfoStream);
            writer.EndWriteEntry();

            InvokeInfo("end backup storage", Tenant.TenantAlias);
        }
예제 #10
0
        private void DoDumpStorage(IDataWriteOperator writer, IReadOnlyList <BackupFileInfo> files)
        {
            Logger.Debug("begin backup storage");

            var dir    = Path.GetDirectoryName(BackupFilePath);
            var subDir = Path.Combine(dir, Path.GetFileNameWithoutExtension(BackupFilePath));

            for (var i = 0; i < files.Count; i += TasksLimit)
            {
                var storageDir = Path.Combine(subDir, KeyHelper.GetStorage());

                if (!Directory.Exists(storageDir))
                {
                    Directory.CreateDirectory(storageDir);
                }

                var tasks = new List <Task>(TasksLimit);
                for (var j = 0; j < TasksLimit && i + j < files.Count; j++)
                {
                    var t = files[i + j];
                    tasks.Add(Task.Run(() => DoDumpFile(t, storageDir)));
                }

                Task.WaitAll(tasks.ToArray());

                ArchiveDir(writer, subDir);

                Directory.Delete(storageDir, true);
            }

            var restoreInfoXml = new XElement("storage_restore", files.Select(file => (object)file.ToXElement()).ToArray());

            var tmpPath = Path.Combine(subDir, KeyHelper.GetStorageRestoreInfoZipKey());

            Directory.CreateDirectory(Path.GetDirectoryName(tmpPath));

            using (var tmpFile = File.OpenWrite(tmpPath))
            {
                restoreInfoXml.WriteTo(tmpFile);
            }

            writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath);
            File.Delete(tmpPath);
            SetStepCompleted();

            Directory.Delete(subDir, true);

            Logger.Debug("end backup storage");
        }
예제 #11
0
 private void ArchiveDir(IDataWriteOperator writer, string subDir)
 {
     Logger.DebugFormat("archive dir start {0}", subDir);
     foreach (var enumerateFile in Directory.EnumerateFiles(subDir, "*", SearchOption.AllDirectories))
     {
         var f = enumerateFile;
         if (!WorkContext.IsMono && enumerateFile.Length > MaxLength)
         {
             f = @"\\?\" + f;
         }
         writer.WriteEntry(enumerateFile.Substring(subDir.Length), f);
         File.Delete(f);
         SetStepCompleted();
     }
     Logger.DebugFormat("archive dir end {0}", subDir);
 }
예제 #12
0
 private void ArchiveDir(IDataWriteOperator writer, string subDir)
 {
     Logger.DebugFormat("archive dir start {0}", subDir);
     foreach (var enumerateFile in Directory.EnumerateFiles(subDir, "*", SearchOption.AllDirectories))
     {
         var f = enumerateFile;
         if (!WorkContext.IsMono && enumerateFile.Length > MaxLength)
         {
             f = @"\\?\" + f;
         }
         using (var tmpFile = new FileStream(f, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose))
         {
             writer.WriteEntry(enumerateFile.Substring(subDir.Length), tmpFile);
         }
         SetStepCompleted();
     }
     Logger.DebugFormat("archive dir end {0}", subDir);
 }
예제 #13
0
        private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module)
        {
            InvokeInfo("begin saving data for module {0}", module.ModuleName);
            int tablesCount = module.Tables.Count();
            int tablesProcessed = 0;
            using (var connection = dbFactory.OpenConnection(module.ConnectionStringName))
            {
                foreach (var table in module.Tables)
                {
                    InvokeInfo("begin saving table {0}", table.Name);
                    using (var data = new DataTable(table.Name))
                    {
                        ActionInvoker.Try(
                            state =>
                                {
                                    data.Clear();
                                    var t = (TableInfo)state;
                                    var dataAdapter = dbFactory.CreateDataAdapter(module.ConnectionStringName);
                                    dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(600);
                                    ((DbDataAdapter)dataAdapter).Fill(data);

                                },
                            table,
                            maxAttempts: 5,
                            onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); },
                            onAttemptFailure: error => InvokeWarning("backup attempt failure: {0}", error));

                        foreach (var col in data.Columns.Cast<DataColumn>().Where(col => col.DataType == typeof(DateTime)))
                        {
                            col.DateTimeMode = DataSetDateTime.Unspecified;
                        }

                        var stream = writer.BeginWriteEntry(KeyHelper.GetTableZipKey(module, data.TableName));
                        data.WriteXml(stream, XmlWriteMode.WriteSchema);
                        writer.EndWriteEntry();
                        data.Clear();
                    }

                    SetStepProgress((int)((++tablesProcessed*100)/(double)tablesCount));
                }
            }
            InvokeInfo("end saving data for module {0}", module.ModuleName);
        }
예제 #14
0
        private void DoBackupModule(IDataWriteOperator writer, DbFactory dbFactory, IModuleSpecifics module)
        {
            InvokeInfo("begin saving data for module {0}", module.ModuleName);
            int tablesCount     = module.Tables.Count();
            int tablesProcessed = 0;

            using (var connection = dbFactory.OpenConnection(module.ConnectionStringName))
            {
                foreach (var table in module.Tables)
                {
                    InvokeInfo("begin saving table {0}", table.Name);
                    using (var data = new DataTable(table.Name))
                    {
                        ActionInvoker.Try(
                            state =>
                        {
                            data.Clear();
                            var t                     = (TableInfo)state;
                            var dataAdapter           = dbFactory.CreateDataAdapter(module.ConnectionStringName);
                            dataAdapter.SelectCommand = module.CreateSelectCommand(connection.Fix(), Tenant.TenantId, t).WithTimeout(600);
                            ((DbDataAdapter)dataAdapter).Fill(data);
                        },
                            table,
                            maxAttempts: 5,
                            onFailure: error => { throw ThrowHelper.CantBackupTable(table.Name, error); },
                            onAttemptFailure: error => InvokeWarning("backup attempt failure: {0}", error));

                        foreach (var col in data.Columns.Cast <DataColumn>().Where(col => col.DataType == typeof(DateTime)))
                        {
                            col.DateTimeMode = DataSetDateTime.Unspecified;
                        }

                        var stream = writer.BeginWriteEntry(KeyHelper.GetTableZipKey(module, data.TableName));
                        data.WriteXml(stream, XmlWriteMode.WriteSchema);
                        writer.EndWriteEntry();
                        data.Clear();
                    }

                    SetStepProgress((int)((++tablesProcessed * 100) / (double)tablesCount));
                }
            }
            InvokeInfo("end saving data for module {0}", module.ModuleName);
        }
예제 #15
0
        private void DoBackupStorage(IDataWriteOperator writer)
        {
            InvokeInfo("begin backup storage", Tenant.TenantAlias);
            var fileGroups = GetFilesToProcess().GroupBy(file => file.Module).ToList();
            int groupsProcessed = 0;
            foreach (var group in fileGroups)
            {
                IDataStore storage = StorageFactory.GetStorage(ConfigPath, Tenant.TenantId.ToString(), group.Key, null, null);
                foreach (BackupFileInfo file in group)
                {
                    Stream stream = writer.BeginWriteEntry(KeyHelper.GetFileZipKey(file));

                    int offset = 0;
                    ActionInvoker.Try(state =>
                        {
                            var f = (BackupFileInfo)state;
                            using (var fileStream = storage.GetReadStream(f.Domain, f.Path, offset))
                            {
                                var buffer = new byte[2048];
                                int readed;
                                while ((readed = fileStream.Read(buffer, 0, buffer.Length)) > 0)
                                {
                                    stream.Write(buffer, 0, readed);
                                    offset += readed;
                                }
                            }
                        }, file, 5, error => InvokeWarning("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));

                    writer.EndWriteEntry();
                }
                SetStepProgress((int)(++groupsProcessed*100/(double)fileGroups.Count));
            }

            if (fileGroups.Count == 0)
                SetStepCompleted();

            var restoreInfoXml = new XElement(
                "storage_restore",
                fileGroups
                    .SelectMany(group => group.Select(file => (object)file.ToXElement()))
                    .ToArray());

            Stream restoreInfoStream = writer.BeginWriteEntry(KeyHelper.GetStorageRestoreInfoZipKey());
            restoreInfoXml.WriteTo(restoreInfoStream);
            writer.EndWriteEntry();

            InvokeInfo("end backup storage", Tenant.TenantAlias);
        }
예제 #16
0
        private List<XElement> BackupDatabase(int tenant, ConnectionStringSettings connectionString, IDataWriteOperator writer)
        {
            var xml = new List<XElement>();
            var errors = 0;
            var timeout = TimeSpan.FromSeconds(1);

            using (var dbHelper = new DbHelper(tenant, connectionString))
            {
                var tables = dbHelper.GetTables();
                for (int i = 0; i < tables.Count; i++)
                {
                    var table = tables[i];
                    OnProgressChanged(table, (int)(i / (double)tables.Count * 100));

                    xml.Add(new XElement(table));
                    var stream = writer.BeginWriteEntry(string.Format("{0}\\{1}\\{2}", Name, connectionString.Name, table).ToLower());
                    DataTable dataTable = null;
                    while (true)
                    {
                        try
                        {
                            dataTable = dbHelper.GetTable(table);
                            break;
                        }
                        catch
                        {
                            errors++;
                            if (20 < errors) throw;
                            Thread.Sleep(timeout);
                        }
                    }
                    foreach (DataColumn c in dataTable.Columns)
                    {
                        if (c.DataType == typeof(DateTime)) c.DateTimeMode = DataSetDateTime.Unspecified;
                    }
                    dataTable.WriteXml(stream, XmlWriteMode.WriteSchema);
                    writer.EndWriteEntry();
                }
            }
            return xml;
        }
예제 #17
0
        private void DumpTableData(string t, string dir, int count, string dbName, IDataWriteOperator writer)
        {
            try
            {
                if (count == 0)
                {
                    Logger.DebugFormat("dump table data stop {0}", t);
                    SetStepCompleted(2);
                    return;
                }

                Logger.DebugFormat("dump table data start {0}", t);
                var    searchWithPrimary = false;
                string primaryIndex;
                int    primaryIndexStep  = 0;
                int    primaryIndexStart = 0;

                List <string> columns;
                using (var dbManager = DbManager.FromHttpContext(dbName, 100000))
                {
                    var columnsData = dbManager.ExecuteList(string.Format("SHOW COLUMNS FROM `{0}`;", t));
                    columns = columnsData
                              .Select(r => "`" + Convert.ToString(r[0]) + "`")
                              .ToList();

                    primaryIndex = dbManager
                                   .ExecuteList(
                        new SqlQuery("information_schema.`COLUMNS`")
                        .Select("COLUMN_NAME")
                        .Where("TABLE_SCHEMA", dbManager.Connection.Database)
                        .Where("TABLE_NAME", t)
                        .Where("COLUMN_KEY", "PRI")
                        .Where("DATA_TYPE", "int"))
                                   .ConvertAll(r => Convert.ToString(r[0]))
                                   .FirstOrDefault();

                    var isLeft = dbManager.ExecuteList(string.Format("SHOW INDEXES FROM {0} WHERE COLUMN_NAME='{1}' AND seq_in_index=1", t, primaryIndex));

                    searchWithPrimary = isLeft.Count == 1;

                    if (searchWithPrimary)
                    {
                        var minMax = dbManager
                                     .ExecuteList(new SqlQuery(t).SelectMax(primaryIndex).SelectMin(primaryIndex))
                                     .ConvertAll(r => new Tuple <int, int>(Convert.ToInt32(r[0]), Convert.ToInt32(r[1])))
                                     .FirstOrDefault();

                        primaryIndexStart = minMax.Item2;
                        primaryIndexStep  = (minMax.Item1 - minMax.Item2) / count;

                        if (primaryIndexStep < Limit)
                        {
                            primaryIndexStep = Limit;
                        }
                    }
                }

                var path   = Path.Combine(dir, t);
                var offset = 0;

                do
                {
                    List <object[]> result;

                    if (searchWithPrimary)
                    {
                        result             = GetDataWithPrimary(t, columns, primaryIndex, primaryIndexStart, primaryIndexStep, dbName);
                        primaryIndexStart += primaryIndexStep;
                    }
                    else
                    {
                        result = GetData(t, columns, offset, dbName);
                    }

                    offset += Limit;

                    var resultCount = result.Count;

                    if (resultCount == 0)
                    {
                        break;
                    }

                    SaveToFile(path, t, columns, result);
                } while (true);

                SetStepCompleted();
                Logger.DebugFormat("dump table data stop {0}", t);
            }
            catch (Exception e)
            {
                Logger.Error(e);
                throw;
            }
        }
예제 #18
0
 public IEnumerable<XElement> GetElements(int tenant, string[] configs, IDataWriteOperator writer)
 {
     return null;
 }
예제 #19
0
        private List <XElement> BackupDatabase(int tenant, ConnectionStringSettings connectionString, IDataWriteOperator writer)
        {
            var xml     = new List <XElement>();
            var errors  = 0;
            var timeout = TimeSpan.FromSeconds(1);
            var tables  = dbHelper.GetTables();

            for (var i = 0; i < tables.Count; i++)
            {
                var table = tables[i];
                OnProgressChanged(table, (int)(i / (double)tables.Count * 100));

                if (processedTables.Contains(table, StringComparer.InvariantCultureIgnoreCase))
                {
                    continue;
                }

                xml.Add(new XElement(table));
                DataTable dataTable = null;
                while (true)
                {
                    try
                    {
                        dataTable = dbHelper.GetTable(table, tenant);
                        break;
                    }
                    catch
                    {
                        errors++;
                        if (20 < errors)
                        {
                            throw;
                        }
                        Thread.Sleep(timeout);
                    }
                }
                foreach (DataColumn c in dataTable.Columns)
                {
                    if (c.DataType == typeof(DateTime))
                    {
                        c.DateTimeMode = DataSetDateTime.Unspecified;
                    }
                }

                var tmp = Path.GetTempFileName();
                using (var file = File.OpenWrite(tmp))
                {
                    dataTable.WriteXml(file, XmlWriteMode.WriteSchema);
                }

                writer.WriteEntry(string.Format("{0}\\{1}\\{2}", Name, connectionString.Name, table).ToLower(), tmp);
                File.Delete(tmp);

                processedTables.Add(table);
            }
            return(xml);
        }
예제 #20
0
        private void DoDump(IDataWriteOperator writer)
        {
            var tmp = Path.GetTempFileName();

            File.AppendAllText(tmp, true.ToString());
            writer.WriteEntry(KeyHelper.GetDumpKey(), tmp);

            List <string> tables;
            var           files = new List <BackupFileInfo>();

            using (var dbManager = new DbManager("default", 100000))
            {
                tables = dbManager.ExecuteList("show tables;").Select(r => Convert.ToString(r[0])).ToList();
            }

            var stepscount = tables.Count * 4; // (schema + data) * (dump + zip)

            if (ProcessStorage)
            {
                var tenants = CoreContext.TenantManager.GetTenants(false).Select(r => r.TenantId);
                foreach (var t in tenants)
                {
                    files.AddRange(GetFiles(t));
                }
                stepscount += files.Count * 2 + 1;
                Logger.Debug("files:" + files.Count);
            }

            SetStepsCount(stepscount);

            var excluded = ModuleProvider.AllModules.Where(r => IgnoredModules.Contains(r.ModuleName)).SelectMany(r => r.Tables).Select(r => r.Name).ToList();

            excluded.AddRange(IgnoredTables);
            excluded.Add("res_");

            var dir       = Path.GetDirectoryName(BackupFilePath);
            var subDir    = Path.Combine(dir, Path.GetFileNameWithoutExtension(BackupFilePath));
            var schemeDir = Path.Combine(subDir, KeyHelper.GetDatabaseSchema());
            var dataDir   = Path.Combine(subDir, KeyHelper.GetDatabaseData());

            if (!Directory.Exists(schemeDir))
            {
                Directory.CreateDirectory(schemeDir);
            }
            if (!Directory.Exists(dataDir))
            {
                Directory.CreateDirectory(dataDir);
            }

            var dict = tables.ToDictionary(t => t, SelectCount);

            tables.Sort((pair1, pair2) => dict[pair1].CompareTo(dict[pair2]));

            for (var i = 0; i < tables.Count; i += TasksLimit)
            {
                var tasks = new List <Task>(TasksLimit * 2);
                for (var j = 0; j < TasksLimit && i + j < tables.Count; j++)
                {
                    var t = tables[i + j];
                    tasks.Add(Task.Run(() => DumpTableScheme(t, schemeDir)));
                    if (!excluded.Any(t.StartsWith))
                    {
                        tasks.Add(Task.Run(() => DumpTableData(t, dataDir, dict[t])));
                    }
                    else
                    {
                        SetStepCompleted(2);
                    }
                }

                Task.WaitAll(tasks.ToArray());

                ArchiveDir(writer, subDir);
            }

            Logger.DebugFormat("dir remove start {0}", subDir);
            Directory.Delete(subDir, true);
            Logger.DebugFormat("dir remove end {0}", subDir);

            if (ProcessStorage)
            {
                DoDumpStorage(writer, files);
            }
        }
예제 #21
0
        private void DoDump(IDataWriteOperator writer, string dbName, List <string> tables)
        {
            var excluded = ModuleProvider.AllModules.Where(r => IgnoredModules.Contains(r.ModuleName)).SelectMany(r => r.Tables).Select(r => r.Name).ToList();

            excluded.AddRange(IgnoredTables);
            excluded.Add("res_");

            var dir       = Path.GetDirectoryName(BackupFilePath);
            var subDir    = Path.Combine(dir, Path.GetFileNameWithoutExtension(BackupFilePath));
            var schemeDir = "";
            var dataDir   = "";

            if (dbName == "default")
            {
                schemeDir = Path.Combine(subDir, KeyHelper.GetDatabaseSchema());
                dataDir   = Path.Combine(subDir, KeyHelper.GetDatabaseData());
            }
            else
            {
                schemeDir = Path.Combine(subDir, dbName, KeyHelper.GetDatabaseSchema());
                dataDir   = Path.Combine(subDir, dbName, KeyHelper.GetDatabaseData());
            }

            if (!Directory.Exists(schemeDir))
            {
                Directory.CreateDirectory(schemeDir);
            }
            if (!Directory.Exists(dataDir))
            {
                Directory.CreateDirectory(dataDir);
            }

            var dict = new Dictionary <string, int>();

            foreach (var table in tables)
            {
                dict.Add(table, SelectCount(table, dbName));
            }
            tables.Sort((pair1, pair2) => dict[pair1].CompareTo(dict[pair2]));

            for (var i = 0; i < tables.Count; i += TasksLimit)
            {
                var tasks = new List <Task>(TasksLimit * 2);
                for (var j = 0; j < TasksLimit && i + j < tables.Count; j++)
                {
                    var t = tables[i + j];
                    tasks.Add(Task.Run(() => DumpTableScheme(t, schemeDir, dbName)));
                    if (!excluded.Any(t.StartsWith))
                    {
                        tasks.Add(Task.Run(() => DumpTableData(t, dataDir, dict[t], dbName, writer)));
                    }
                    else
                    {
                        SetStepCompleted(2);
                    }
                }

                Task.WaitAll(tasks.ToArray());

                ArchiveDir(writer, subDir);
            }
        }
예제 #22
0
        private List <XElement> BackupDatabase(int tenant, ConnectionStringSettings connectionString, IDataWriteOperator writer)
        {
            var xml     = new List <XElement>();
            var errors  = 0;
            var timeout = TimeSpan.FromSeconds(1);

            using (var dbHelper = new DbHelper(tenant, connectionString))
            {
                var tables = dbHelper.GetTables();
                for (int i = 0; i < tables.Count; i++)
                {
                    var table = tables[i];
                    OnProgressChanged(table, (int)(i / (double)tables.Count * 100));

                    xml.Add(new XElement(table));
                    var       stream    = writer.BeginWriteEntry(string.Format("{0}\\{1}\\{2}", Name, connectionString.Name, table).ToLower());
                    DataTable dataTable = null;
                    while (true)
                    {
                        try
                        {
                            dataTable = dbHelper.GetTable(table);
                            break;
                        }
                        catch
                        {
                            errors++;
                            if (20 < errors)
                            {
                                throw;
                            }
                            Thread.Sleep(timeout);
                        }
                    }
                    foreach (DataColumn c in dataTable.Columns)
                    {
                        if (c.DataType == typeof(DateTime))
                        {
                            c.DateTimeMode = DataSetDateTime.Unspecified;
                        }
                    }
                    dataTable.WriteXml(stream, XmlWriteMode.WriteSchema);
                    writer.EndWriteEntry();
                }
            }
            return(xml);
        }
예제 #23
0
        private void DoBackupStorage(IDataWriteOperator writer, DbFactory dbFactory)
        {
            Logger.Debug("begin backup storage");

            var files   = GetFilesToProcess();
            var exclude = new List <string>();

            using (var db = dbFactory.OpenConnection())
                using (var command = db.CreateCommand())
                {
                    command.CommandText = "select storage_path from backup_backup where tenant_id = " + TenantId + " and storage_type = 0 and storage_path is not null";
                    using (var reader = command.ExecuteReader())
                    {
                        while (reader.Read())
                        {
                            exclude.Add(reader.GetString(0));
                        }
                    }
                }
            files = files.Where(f => !exclude.Any(e => f.Path.Contains(string.Format("/file_{0}/", e))));

            var fileGroups      = files.GroupBy(file => file.Module).ToList();
            var groupsProcessed = 0;

            foreach (var group in fileGroups)
            {
                var storage = StorageFactory.GetStorage(ConfigPath, TenantId.ToString(), group.Key);
                foreach (var file in group)
                {
                    ActionInvoker.Try(state =>
                    {
                        var f = (BackupFileInfo)state;
                        using (var fileStream = storage.GetReadStream(f.Domain, f.Path))
                        {
                            var tmp = Path.GetTempFileName();
                            try
                            {
                                using (var tmpFile = File.OpenWrite(tmp))
                                {
                                    fileStream.CopyTo(tmpFile);
                                }

                                writer.WriteEntry(KeyHelper.GetFileZipKey(file), tmp);
                            }
                            finally
                            {
                                if (File.Exists(tmp))
                                {
                                    File.Delete(tmp);
                                }
                            }
                        }
                    }, file, 5, error => Logger.Warn("can't backup file ({0}:{1}): {2}", file.Module, file.Path, error));
                }
                SetCurrentStepProgress((int)(++groupsProcessed * 100 / (double)fileGroups.Count));
            }

            if (fileGroups.Count == 0)
            {
                SetStepCompleted();
            }

            var restoreInfoXml = new XElement(
                "storage_restore",
                fileGroups
                .SelectMany(group => group.Select(file => (object)file.ToXElement()))
                .ToArray());

            var tmpPath = Path.GetTempFileName();

            using (var tmpFile = File.OpenWrite(tmpPath))
            {
                restoreInfoXml.WriteTo(tmpFile);
            }

            writer.WriteEntry(KeyHelper.GetStorageRestoreInfoZipKey(), tmpPath);
            File.Delete(tmpPath);


            Logger.Debug("end backup storage");
        }
 public IEnumerable <XElement> GetElements(int tenant, string[] configs, IDataWriteOperator writer)
 {
     return(null);
 }
예제 #25
0
        private void DoDump(IDataWriteOperator writer)
        {
            Dictionary <string, List <string> > databases = new Dictionary <string, List <string> >();

            using (var dbManager = DbManager.FromHttpContext("default", 100000))
            {
                dbManager.ExecuteList("select id, connection_string from mail_server_server").ForEach((r =>
                {
                    var dbName = GetDbName((int)r[0], JsonConvert.DeserializeObject <Dictionary <string, object> >(Convert.ToString(r[1]))["DbConnection"].ToString());
                    using (var dbManager1 = DbManager.FromHttpContext(dbName, 100000))
                    {
                        var tables = dbManager1.ExecuteList("show tables;").Select(res => Convert.ToString(res[0])).ToList();
                        databases.Add(dbName, tables);
                    }
                }));
            }

            using (var dbManager = DbManager.FromHttpContext("default", 100000))
            {
                var tables = dbManager.ExecuteList("show tables;").Select(res => Convert.ToString(res[0])).ToList();
                databases.Add("default", tables);
            }

            using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(true.ToString())))
            {
                writer.WriteEntry(KeyHelper.GetDumpKey(), stream);
            }

            var files = new List <BackupFileInfo>();

            var stepscount = 0;

            foreach (var db in databases)
            {
                stepscount += db.Value.Count * 4;// (schema + data) * (dump + zip)
            }
            if (ProcessStorage)
            {
                var tenants = CoreContext.TenantManager.GetTenants(false).Select(r => r.TenantId);
                foreach (var t in tenants)
                {
                    files.AddRange(GetFiles(t));
                }
                stepscount += files.Count * 2 + 1;
                Logger.Debug("files:" + files.Count);
            }

            SetStepsCount(stepscount);

            foreach (var db in databases)
            {
                DoDump(writer, db.Key, db.Value);
            }
            var dir    = Path.GetDirectoryName(BackupFilePath);
            var subDir = Path.Combine(dir, Path.GetFileNameWithoutExtension(BackupFilePath));

            Logger.DebugFormat("dir remove start {0}", subDir);
            Directory.Delete(subDir, true);
            Logger.DebugFormat("dir remove end {0}", subDir);

            if (ProcessStorage)
            {
                DoDumpStorage(writer, files);
            }
        }