private static Stream[] CreatePipeline(List <ConfigPair> pipelineConfig, IList <Stream> fileStreams, bool isBackup, IStreamNotification streamNotification, long estimatedTotalBytes) { streamNotification.EstimatedBytes = estimatedTotalBytes; List <Stream> result = new List <Stream>(fileStreams.Count); foreach (Stream fileStream in fileStreams) { Stream topStream = fileStream; if (!isBackup) { topStream = new TrackingStream(topStream, streamNotification); } for (int i = pipelineConfig.Count - 1; i >= 0; i--) { ConfigPair config = pipelineConfig[i]; IBackupTransformer tran = config.TransformationType.GetConstructor(new Type[0]).Invoke(new object[0]) as IBackupTransformer; if (tran == null) { throw new ArgumentException(string.Format("Unable to create pipe component: {0}", config.TransformationType.Name)); } topStream = isBackup ? tran.GetBackupWriter(config.Parameters, topStream) : tran.GetRestoreReader(config.Parameters, topStream); } if (isBackup) { topStream = new TrackingStream(topStream, streamNotification); } result.Add(topStream); } return(result.ToArray()); }
public static void Backup(ConfigPair databaseConfig, List <ConfigPair> pipelineConfig, ConfigPair storageConfig, IUpdateNotification updateNotifier, out List <string> devices) { List <string> returndevices; BackupOrRestore(1, storageConfig, databaseConfig, pipelineConfig, updateNotifier, out returndevices); devices = returndevices; }
public static void WriteHeaderFilelist(ConfigPair databaseConfig, ConfigPair storageConfig, IEnumerable <string> devices) { /* * 12 --2014 supported * 11 --2012 supported * 10 --2008/R2 supported * 9 --2005 supported * 8 --2000 NOT supported * 7 --7.0 NOT supported */ string instanceName = null; if (databaseConfig.Parameters.ContainsKey("instancename")) { instanceName = databaseConfig.Parameters["instancename"][0]; } if (instanceName != null) { instanceName = instanceName.Trim(); } string clusterNetworkName = null; if (databaseConfig.Parameters.ContainsKey("ClusterNetworkName")) { clusterNetworkName = databaseConfig.Parameters["ClusterNetworkName"][0]; } var serverConnectionName = clusterNetworkName ?? "."; var dataSource = String.IsNullOrEmpty(instanceName) ? serverConnectionName : String.Format(@"{0}\{1}", serverConnectionName, instanceName); string connectionString; if (databaseConfig.Parameters.ContainsKey("user")) { connectionString = databaseConfig.Parameters.ContainsKey("password") ? String.Format("Data Source={0};Initial Catalog=master;Integrated Security=False;User ID={1};Password={2};Connect Timeout=2147483647;", dataSource, databaseConfig.Parameters["user"][0], databaseConfig.Parameters["password"][0]) : String.Format("Data Source={0};Initial Catalog=master;Integrated Security=False;User ID={1};Password={2};Connect Timeout=2147483647;", dataSource, databaseConfig.Parameters["user"][0], null); } else { connectionString = String.Format("Data Source={0};Initial Catalog=master;Integrated Security=True;Connect Timeout=2147483647;", dataSource); } int majorVersionNum; using (var cnn = new SqlConnection(connectionString)) { var version = GetVersion(cnn); majorVersionNum = GetMajorVersionNumber(version); } var fileListHeaderOnlyQuery = new StringBuilder(); const string queryCap = @") ) );"; const string headeronly = @"SELECT [name] AS BackupName, [description] AS BackupDescription, CASE WHEN [type] = 'D' THEN 1 WHEN [type] = 'I' THEN 5 WHEN [type] = 'L' THEN 2 WHEN [type] = 'F' THEN 4 WHEN [type] = 'G' THEN 6 WHEN [type] = 'P' THEN 7 WHEN [type] = 'Q' THEN 8 ELSE NULL END AS BackupType, expiration_date AS ExpirationDate, 0 AS Compressed, position AS Position, 2 AS DeviceType, [user_name] AS UserName, [server_name] AS ServerName, [database_name] AS DatabaseName, database_version AS DatabaseVersion, database_creation_date AS DatabaseCreationDate, backup_size AS BackupSize, first_lsn AS FirstLSN, last_lsn AS LastLSN, checkpoint_lsn AS CheckpointLSN, database_backup_lsn AS DatabaseBackupLSN, backup_start_date AS BackupStartDate, backup_finish_date AS BackupFinishDate, sort_order AS SortOrder, code_page AS CodePage, unicode_locale AS UnicodeLocaleid, unicode_compare_style AS UnicodeComparisionStyle, compatibility_level AS CompatibilityLevel, software_vendor_id AS SoftwareVendorid, software_major_version AS SoftwareVersionMajor, software_minor_version AS SoftwareVersionMinor, software_build_version AS SoftwareVersionBuild, machine_name AS MachineName, flags AS Flags, drs.database_guid AS BindingID, drs.recovery_fork_guid AS RecoveryForkID, collation_name AS Collation, bs.family_guid AS FamilyGUID, has_bulk_logged_data AS HasBulkLoggedData, is_snapshot AS IsSnapshot, is_readonly AS IsReadonly, is_single_user AS IsSingleUser, has_backup_checksums AS HasBackupChecksums, is_damaged AS IsDamaged, begins_log_chain AS BeginsLogChain, has_incomplete_metadata AS HasIncompleteMetaData, is_force_offline AS IsForceOffline, is_copy_only AS IsCopyOnly, bs.first_recovery_fork_guid AS FirstRecoveryForkID, bs.fork_point_lsn AS ForPointLSN, recovery_model AS RecoveryModel, differential_base_lsn AS DifferentialBaseLSN, differential_base_guid AS DifferentialBaseGUID, CASE WHEN [type] = 'D' THEN 'DATABASE' WHEN [type] = 'I' THEN 'DATABASE DIFFERENTIAL' WHEN [type] = 'L' THEN 'TRANSACTION LOG' WHEN [type] = 'F' THEN 'FILE OR FILEGROUP' WHEN [type] = 'G' THEN 'FILE DIFFERENTIAL PARTIAL' WHEN [type] = 'P' THEN 'PARTIAL DIFFERENTIAL' WHEN [type] = 'Q' THEN 'PARTIAL DIFFERENTIAL' ELSE NULL END AS BackupSetDescription, backup_set_uuid AS BackupSetGUID"; const string headeronly2008 = @", compressed_backup_size AS CompressedBackupSize"; const string headeronly2012 = @", compressed_backup_size AS CompressedBackupSize, 0 AS Containment "; const string headeronlytail = @" FROM msdb.dbo.backupset bs INNER JOIN msdb.sys.database_recovery_status drs ON bs.family_guid = drs.family_guid AND bs.database_guid = drs.database_guid where bs.backup_set_id in( select backup_set_id from msdb.dbo.backupset where media_set_id in ( select distinct media_set_id from msdb.dbo.backupmediafamily where physical_device_name in(" ; const string fileListOnly = @"SELECT bf.logical_name AS LogicalName, bf.physical_name AS PhysicalName, bf.file_type AS [Type], bf.[filegroup_name] AS FileGroupName, bf.[file_size] AS Size, 35184372080640 AS MaxSize, bf.file_number AS FileID, bf.create_lsn AS CreateLSN, bf.drop_lsn AS DropLSN, bf.file_guid AS UniqueID, bf.read_only_lsn AS ReadOnlyLSN, bf.read_write_lsn AS ReadWriteLSN, bf.backup_size AS BackupSizeInBytes, bf.source_file_block_size AS SourceBlockSize, bfg.[filegroup_id] AS FileGroupID, NULL AS logLogGroupGUID, bf.differential_base_lsn AS DifferentialBaseLSN, bf.differential_base_guid AS DifferentialBaseGUID, bf.is_readonly AS IsReadOnly, bf.is_present AS IsPresent"; const string filelistonly2008 = @", NULL AS TDEThumbprint "; const string filelistonlytail = @" FROM msdb.dbo.backupfile bf left outer join msdb.dbo.backupfilegroup bfg on bf.backup_set_id = bfg.backup_set_id and bf.filegroup_guid = bf.filegroup_guid where bf.backup_set_id in ( select backup_set_id from msdb.dbo.backupset where media_set_id in ( select distinct media_set_id from msdb.dbo.backupmediafamily where physical_device_name in(" ; var metaDataPath = storageConfig.Parameters["path"][0] + ".hfl"; var deviceList = new StringBuilder(); foreach (var d in devices) { deviceList.Append("'"); deviceList.Append(d); deviceList.Append("',"); } deviceList.Remove(deviceList.Length - 1, 1); fileListHeaderOnlyQuery.Append(headeronly); switch (majorVersionNum) { case 12: fileListHeaderOnlyQuery.Append(headeronly2012); break; case 11: fileListHeaderOnlyQuery.Append(headeronly2012); break; case 10: fileListHeaderOnlyQuery.Append(headeronly2008); break; } fileListHeaderOnlyQuery.Append(headeronlytail); fileListHeaderOnlyQuery.Append(deviceList); fileListHeaderOnlyQuery.Append(queryCap); fileListHeaderOnlyQuery.Append(fileListOnly); switch (majorVersionNum) { case 12: fileListHeaderOnlyQuery.Append(filelistonly2008); break; case 11: fileListHeaderOnlyQuery.Append(filelistonly2008); break; case 10: fileListHeaderOnlyQuery.Append(filelistonly2008); break; } fileListHeaderOnlyQuery.Append(filelistonlytail); fileListHeaderOnlyQuery.Append(deviceList); fileListHeaderOnlyQuery.Append(queryCap); var headerFileList = new DataSet(); /* * we only support sql server 2005 and above for meta-data right now */ //TODO: add support for 2000 and 7.0 if (majorVersionNum > 8) { using (var mCnn = new SqlConnection(connectionString)) { var SelectCommand = new SqlCommand(fileListHeaderOnlyQuery.ToString(), mCnn); SelectCommand.CommandTimeout = 0; var adapter = new SqlDataAdapter(SelectCommand); mCnn.Open(); adapter.Fill(headerFileList); adapter.Dispose(); } using (var fs = new FileStream(metaDataPath, FileMode.Create)) { headerFileList.RemotingFormat = SerializationFormat.Binary; var bFormat = new BinaryFormatter(); bFormat.Serialize(fs, headerFileList); } } headerFileList.Dispose(); }
private static void BackupOrRestore(bool isBackup, ConfigPair storageConfig, ConfigPair databaseConfig, List <ConfigPair> pipelineConfig, IUpdateNotification updateNotifier) { string deviceSetName = Guid.NewGuid().ToString(); IBackupStorage storage = storageConfig.TransformationType.GetConstructor(new Type[0]).Invoke(new object[0]) as IBackupStorage; IBackupDatabase databaseComp = databaseConfig.TransformationType.GetConstructor(new Type[0]).Invoke(new object[0]) as IBackupDatabase; try { int numDevices = storage.GetNumberOfDevices(storageConfig.Parameters); string instanceName = databaseComp.GetInstanceName(databaseConfig.Parameters); string clusterNetworkName = databaseComp.GetClusterNetworkName(databaseConfig.Parameters); SqlPlatform sqlPlatform = VirtualBackupDeviceFactory.DiscoverSqlPlatform(instanceName, clusterNetworkName); //NotifyWhenReady notifyWhenReady = new NotifyWhenReady(deviceName, isBackup);)) using (IVirtualDeviceSet deviceSet = VirtualBackupDeviceFactory.NewVirtualDeviceSet(sqlPlatform)) { using (SqlThread sql = new SqlThread()) { bool sqlStarted = false; bool sqlFinished = false; ParallelExecutionException exceptions = new ParallelExecutionException(); try { IStreamNotification streamNotification = new InternalStreamNotification(updateNotifier); long estimatedTotalBytes; List <string> deviceNames = sql.PreConnect(clusterNetworkName, instanceName, deviceSetName, numDevices, databaseComp, databaseConfig.Parameters, isBackup, updateNotifier, out estimatedTotalBytes); using (DisposableList <Stream> fileStreams = new DisposableList <Stream>(isBackup ? storage.GetBackupWriter(storageConfig.Parameters) : storage.GetRestoreReader(storageConfig.Parameters, out estimatedTotalBytes))) using (DisposableList <Stream> topOfPilelines = new DisposableList <Stream>(CreatePipeline(pipelineConfig, fileStreams, isBackup, streamNotification, estimatedTotalBytes))) { VirtualDeviceSetConfig config = new VirtualDeviceSetConfig(); config.Features = FeatureSet.PipeLike; config.DeviceCount = (uint)topOfPilelines.Count; deviceSet.CreateEx(instanceName, deviceSetName, config); sql.BeginExecute(); sqlStarted = true; deviceSet.GetConfiguration(TimeSpan.FromMinutes(1)); List <IVirtualDevice> devices = new List <IVirtualDevice>(); foreach (string devName in deviceNames) { devices.Add(deviceSet.OpenDevice(devName)); } using (DisposableList <DeviceThread> threads = new DisposableList <DeviceThread>(devices.Count)) { for (int i = 0; i < devices.Count; i++) { DeviceThread dThread = new DeviceThread(); threads.Add(dThread); dThread.Initialize(isBackup, topOfPilelines[i], devices[i], deviceSet); } foreach (DeviceThread dThread in threads) { dThread.BeginCopy(); } updateNotifier.OnStart(); //Console.WriteLine(string.Format("{0} started", isBackup ? "Backup" : "Restore")); Exception sqlE = sql.EndExecute(); sqlFinished = true; if (sqlE != null) { exceptions.Exceptions.Add(sqlE); } foreach (DeviceThread dThread in threads) { Exception devE = dThread.EndCopy(); if (devE != null) { exceptions.Exceptions.Add(devE); } } } } } catch (Exception e) { exceptions.Exceptions.Add(e); } finally { if (sqlStarted && !sqlFinished) { Exception sqlE = sql.EndExecute(); sqlFinished = true; if (sqlE != null) { exceptions.Exceptions.Add(sqlE); } } } if (exceptions.HasExceptions) { throw exceptions; } } } } catch { storage.CleanupOnAbort(); throw; } }
public static void Restore(ConfigPair storageConfig, List <ConfigPair> pipelineConfig, ConfigPair databaseConfig, IUpdateNotification updateNotifier) { BackupOrRestore(false, storageConfig, databaseConfig, pipelineConfig, updateNotifier); }
private static void BackupOrRestore(int commandType, ConfigPair storageConfig, ConfigPair databaseConfig, List <ConfigPair> pipelineConfig, IUpdateNotification updateNotifier, out List <string> returndevices) { var isBackup = commandType == 1; var deviceSetName = Guid.NewGuid().ToString(); var constructorInfo = storageConfig.TransformationType.GetConstructor(new Type[0]); if (constructorInfo == null) { throw new ArgumentException("Unable to transformation type"); } var storage = constructorInfo.Invoke(new object[0]) as IBackupStorage; var constructor = databaseConfig.TransformationType.GetConstructor(new Type[0]); if (constructor == null) { throw new ArgumentException("Unable to transformation type"); } var databaseComp = constructor.Invoke(new object[0]) as IBackupDatabase; try { if (storage == null) { returndevices = null; return; } var numDevices = storage.GetNumberOfDevices(storageConfig.Parameters); //get instance name e.g. myserver\myinstance if (databaseComp == null) { returndevices = null; return; } var instanceName = databaseComp.GetInstanceName(databaseConfig.Parameters); //get the cluster networking name usually diffrent than the currently running node name var clusterNetworkName = databaseComp.GetClusterNetworkName(databaseConfig.Parameters); //get bit version of our platform x86,x64 or IA64 var sqlPlatform = databaseConfig.Parameters.ContainsKey("user") ? VirtualBackupDeviceFactory.DiscoverSqlPlatform(instanceName, clusterNetworkName, databaseConfig.Parameters["user"][0], databaseConfig.Parameters.ContainsKey("password") ? databaseConfig.Parameters["password"][0] : null) : VirtualBackupDeviceFactory.DiscoverSqlPlatform(instanceName, clusterNetworkName, null, null); //NotifyWhenReady notifyWhenReady = new NotifyWhenReady(deviceName, isBackup);)) using (var deviceSet = VirtualBackupDeviceFactory.NewVirtualDeviceSet(sqlPlatform)) { //Since it is possible to have multiple backup/restore targets we encapsulate each call to a device in its own thread using (var sql = new SqlThread()) { var sqlStarted = false; var sqlFinished = false; returndevices = new List <string>(); var exceptions = new ParallelExecutionException(); try { IStreamNotification streamNotification = new InternalStreamNotification(updateNotifier); long estimatedTotalBytes; var deviceNames = sql.PreConnect(clusterNetworkName, instanceName, deviceSetName, numDevices, databaseComp, databaseConfig.Parameters, commandType, updateNotifier, out estimatedTotalBytes); using ( var fileStreams = new DisposableList <Stream>(isBackup ? storage.GetBackupWriter(storageConfig.Parameters, estimatedTotalBytes) : storage.GetRestoreReader(storageConfig.Parameters, out estimatedTotalBytes))) using ( var topOfPilelines = new DisposableList <Stream>(CreatePipeline(pipelineConfig, fileStreams, isBackup, streamNotification, estimatedTotalBytes))) { ReturnByteScale(estimatedTotalBytes, isBackup); var config = new VirtualDeviceSetConfig { Features = FeatureSet.PipeLike, DeviceCount = (uint)topOfPilelines.Count }; deviceSet.CreateEx(instanceName, deviceSetName, config); sql.BeginExecute(); sqlStarted = true; deviceSet.GetConfiguration(TimeSpan.FromMinutes(360)); var devices = deviceNames.Select(deviceSet.OpenDevice).ToList(); returndevices = deviceNames; using (var threads = new DisposableList <DeviceThread>(devices.Count)) { for (var i = 0; i < devices.Count; i++) { var dThread = new DeviceThread(); threads.Add(dThread); dThread.Initialize(isBackup, topOfPilelines[i], devices[i], deviceSet); } foreach (var dThread in threads) { dThread.BeginCopy(); } updateNotifier.OnStart(); var sqlE = sql.EndExecute(); sqlFinished = true; if (sqlE != null) { exceptions.Exceptions.Add(sqlE); } foreach (var devE in threads.Select(dThread => dThread.EndCopy()).Where(devE => devE != null)) { exceptions.Exceptions.Add(devE); } } } } catch (Exception e) { exceptions.Exceptions.Add(e); } finally { if (exceptions.HasExceptions) { throw exceptions; } if (sqlStarted && !sqlFinished) { var sqlE = sql.EndExecute(); if (sqlE != null) { exceptions.Exceptions.Add(sqlE); } } } } } } catch { if (storage != null) { storage.CleanupOnAbort(); } throw; } }
public static void HeaderOnly(ConfigPair storageConfig, List <ConfigPair> pipelineConfig, ConfigPair databaseConfig, IUpdateNotification updateNotifier) { List <string> returndevices; BackupOrRestore(4, storageConfig, databaseConfig, pipelineConfig, updateNotifier, out returndevices); }