private static Stream[] CreatePipeline(List <ConfigPair> pipelineConfig, IList <Stream> fileStreams, bool isBackup, IStreamNotification streamNotification, long estimatedTotalBytes)
{
streamNotification.EstimatedBytes = estimatedTotalBytes;
List <Stream> result = new List <Stream>(fileStreams.Count);
foreach (Stream fileStream in fileStreams)
{
Stream topStream = fileStream;
if (!isBackup)
{
topStream = new TrackingStream(topStream, streamNotification);
}
for (int i = pipelineConfig.Count - 1; i >= 0; i--)
{
ConfigPair config = pipelineConfig[i];
IBackupTransformer tran = config.TransformationType.GetConstructor(new Type[0]).Invoke(new object[0]) as IBackupTransformer;
if (tran == null)
{
throw new ArgumentException(string.Format("Unable to create pipe component: {0}", config.TransformationType.Name));
}
topStream = isBackup ? tran.GetBackupWriter(config.Parameters, topStream) : tran.GetRestoreReader(config.Parameters, topStream);
}
if (isBackup)
{
topStream = new TrackingStream(topStream, streamNotification);
}
result.Add(topStream);
}
return(result.ToArray());
}
public static void Backup(ConfigPair databaseConfig, List <ConfigPair> pipelineConfig, ConfigPair storageConfig, IUpdateNotification updateNotifier, out List <string> devices)
{
List <string> returndevices;
BackupOrRestore(1, storageConfig, databaseConfig, pipelineConfig, updateNotifier, out returndevices);
devices = returndevices;
}
public static void WriteHeaderFilelist(ConfigPair databaseConfig, ConfigPair storageConfig, IEnumerable <string> devices)
{
/*
* 12 --2014 supported
* 11 --2012 supported
* 10 --2008/R2 supported
* 9 --2005 supported
* 8 --2000 NOT supported
* 7 --7.0 NOT supported
*/
string instanceName = null;
if (databaseConfig.Parameters.ContainsKey("instancename"))
{
instanceName = databaseConfig.Parameters["instancename"][0];
}
if (instanceName != null)
{
instanceName = instanceName.Trim();
}
string clusterNetworkName = null;
if (databaseConfig.Parameters.ContainsKey("ClusterNetworkName"))
{
clusterNetworkName = databaseConfig.Parameters["ClusterNetworkName"][0];
}
var serverConnectionName = clusterNetworkName ?? ".";
var dataSource = String.IsNullOrEmpty(instanceName) ? serverConnectionName : String.Format(@"{0}\{1}", serverConnectionName, instanceName);
string connectionString;
if (databaseConfig.Parameters.ContainsKey("user"))
{
connectionString = databaseConfig.Parameters.ContainsKey("password") ? String.Format("Data Source={0};Initial Catalog=master;Integrated Security=False;User ID={1};Password={2};Connect Timeout=2147483647;", dataSource, databaseConfig.Parameters["user"][0], databaseConfig.Parameters["password"][0]) : String.Format("Data Source={0};Initial Catalog=master;Integrated Security=False;User ID={1};Password={2};Connect Timeout=2147483647;", dataSource, databaseConfig.Parameters["user"][0], null);
}
else
{
connectionString = String.Format("Data Source={0};Initial Catalog=master;Integrated Security=True;Connect Timeout=2147483647;", dataSource);
}
int majorVersionNum;
using (var cnn = new SqlConnection(connectionString))
{
var version = GetVersion(cnn);
majorVersionNum = GetMajorVersionNumber(version);
}
var fileListHeaderOnlyQuery = new StringBuilder();
const string queryCap = @")
)
);";
const string headeronly = @"SELECT [name] AS BackupName,
[description] AS BackupDescription,
CASE
WHEN [type] = 'D' THEN 1
WHEN [type] = 'I' THEN 5
WHEN [type] = 'L' THEN 2
WHEN [type] = 'F' THEN 4
WHEN [type] = 'G' THEN 6
WHEN [type] = 'P' THEN 7
WHEN [type] = 'Q' THEN 8
ELSE NULL
END AS BackupType,
expiration_date AS ExpirationDate,
0 AS Compressed,
position AS Position,
2 AS DeviceType,
[user_name] AS UserName,
[server_name] AS ServerName,
[database_name] AS DatabaseName,
database_version AS DatabaseVersion,
database_creation_date AS DatabaseCreationDate,
backup_size AS BackupSize,
first_lsn AS FirstLSN,
last_lsn AS LastLSN,
checkpoint_lsn AS CheckpointLSN,
database_backup_lsn AS DatabaseBackupLSN,
backup_start_date AS BackupStartDate,
backup_finish_date AS BackupFinishDate,
sort_order AS SortOrder,
code_page AS CodePage,
unicode_locale AS UnicodeLocaleid,
unicode_compare_style AS UnicodeComparisionStyle,
compatibility_level AS CompatibilityLevel,
software_vendor_id AS SoftwareVendorid,
software_major_version AS SoftwareVersionMajor,
software_minor_version AS SoftwareVersionMinor,
software_build_version AS SoftwareVersionBuild,
machine_name AS MachineName,
flags AS Flags,
drs.database_guid AS BindingID,
drs.recovery_fork_guid AS RecoveryForkID,
collation_name AS Collation,
bs.family_guid AS FamilyGUID,
has_bulk_logged_data AS HasBulkLoggedData,
is_snapshot AS IsSnapshot,
is_readonly AS IsReadonly,
is_single_user AS IsSingleUser,
has_backup_checksums AS HasBackupChecksums,
is_damaged AS IsDamaged,
begins_log_chain AS BeginsLogChain,
has_incomplete_metadata AS HasIncompleteMetaData,
is_force_offline AS IsForceOffline,
is_copy_only AS IsCopyOnly,
bs.first_recovery_fork_guid AS FirstRecoveryForkID,
bs.fork_point_lsn AS ForPointLSN,
recovery_model AS RecoveryModel,
differential_base_lsn AS DifferentialBaseLSN,
differential_base_guid AS DifferentialBaseGUID,
CASE
WHEN [type] = 'D' THEN 'DATABASE'
WHEN [type] = 'I' THEN 'DATABASE DIFFERENTIAL'
WHEN [type] = 'L' THEN 'TRANSACTION LOG'
WHEN [type] = 'F' THEN 'FILE OR FILEGROUP'
WHEN [type] = 'G' THEN 'FILE DIFFERENTIAL PARTIAL'
WHEN [type] = 'P' THEN 'PARTIAL DIFFERENTIAL'
WHEN [type] = 'Q' THEN 'PARTIAL DIFFERENTIAL'
ELSE NULL
END AS BackupSetDescription,
backup_set_uuid AS BackupSetGUID";
const string headeronly2008 = @",
compressed_backup_size AS CompressedBackupSize";
const string headeronly2012 = @",
compressed_backup_size AS CompressedBackupSize,
0 AS Containment ";
const string headeronlytail = @"
FROM msdb.dbo.backupset bs
INNER JOIN msdb.sys.database_recovery_status drs
ON bs.family_guid = drs.family_guid
AND
bs.database_guid = drs.database_guid
where bs.backup_set_id
in(
select
backup_set_id
from
msdb.dbo.backupset
where
media_set_id in
(
select
distinct media_set_id
from
msdb.dbo.backupmediafamily
where
physical_device_name in(" ;
const string fileListOnly = @"SELECT
bf.logical_name AS LogicalName,
bf.physical_name AS PhysicalName,
bf.file_type AS [Type],
bf.[filegroup_name] AS FileGroupName,
bf.[file_size] AS Size,
35184372080640 AS MaxSize,
bf.file_number AS FileID,
bf.create_lsn AS CreateLSN,
bf.drop_lsn AS DropLSN,
bf.file_guid AS UniqueID,
bf.read_only_lsn AS ReadOnlyLSN,
bf.read_write_lsn AS ReadWriteLSN,
bf.backup_size AS BackupSizeInBytes,
bf.source_file_block_size AS SourceBlockSize,
bfg.[filegroup_id] AS FileGroupID,
NULL AS logLogGroupGUID,
bf.differential_base_lsn AS DifferentialBaseLSN,
bf.differential_base_guid AS DifferentialBaseGUID,
bf.is_readonly AS IsReadOnly,
bf.is_present AS IsPresent";
const string filelistonly2008 = @",
NULL AS TDEThumbprint ";
const string filelistonlytail = @"
FROM
msdb.dbo.backupfile bf
left outer join
msdb.dbo.backupfilegroup bfg
on
bf.backup_set_id = bfg.backup_set_id
and
bf.filegroup_guid = bf.filegroup_guid
where
bf.backup_set_id in
(
select
backup_set_id
from
msdb.dbo.backupset
where
media_set_id in
(
select
distinct media_set_id
from
msdb.dbo.backupmediafamily
where
physical_device_name in(" ;
var metaDataPath = storageConfig.Parameters["path"][0] + ".hfl";
var deviceList = new StringBuilder();
foreach (var d in devices)
{
deviceList.Append("'");
deviceList.Append(d);
deviceList.Append("',");
}
deviceList.Remove(deviceList.Length - 1, 1);
fileListHeaderOnlyQuery.Append(headeronly);
switch (majorVersionNum)
{
case 12:
fileListHeaderOnlyQuery.Append(headeronly2012);
break;
case 11:
fileListHeaderOnlyQuery.Append(headeronly2012);
break;
case 10:
fileListHeaderOnlyQuery.Append(headeronly2008);
break;
}
fileListHeaderOnlyQuery.Append(headeronlytail);
fileListHeaderOnlyQuery.Append(deviceList);
fileListHeaderOnlyQuery.Append(queryCap);
fileListHeaderOnlyQuery.Append(fileListOnly);
switch (majorVersionNum)
{
case 12:
fileListHeaderOnlyQuery.Append(filelistonly2008);
break;
case 11:
fileListHeaderOnlyQuery.Append(filelistonly2008);
break;
case 10:
fileListHeaderOnlyQuery.Append(filelistonly2008);
break;
}
fileListHeaderOnlyQuery.Append(filelistonlytail);
fileListHeaderOnlyQuery.Append(deviceList);
fileListHeaderOnlyQuery.Append(queryCap);
var headerFileList = new DataSet();
/*
* we only support sql server 2005 and above for meta-data right now
*/
//TODO: add support for 2000 and 7.0
if (majorVersionNum > 8)
{
using (var mCnn = new SqlConnection(connectionString))
{
var SelectCommand = new SqlCommand(fileListHeaderOnlyQuery.ToString(), mCnn);
SelectCommand.CommandTimeout = 0;
var adapter = new SqlDataAdapter(SelectCommand);
mCnn.Open();
adapter.Fill(headerFileList);
adapter.Dispose();
}
using (var fs = new FileStream(metaDataPath, FileMode.Create))
{
headerFileList.RemotingFormat = SerializationFormat.Binary;
var bFormat = new BinaryFormatter();
bFormat.Serialize(fs, headerFileList);
}
}
headerFileList.Dispose();
}
private static void BackupOrRestore(bool isBackup, ConfigPair storageConfig, ConfigPair databaseConfig, List <ConfigPair> pipelineConfig, IUpdateNotification updateNotifier)
{
string deviceSetName = Guid.NewGuid().ToString();
IBackupStorage storage = storageConfig.TransformationType.GetConstructor(new Type[0]).Invoke(new object[0]) as IBackupStorage;
IBackupDatabase databaseComp = databaseConfig.TransformationType.GetConstructor(new Type[0]).Invoke(new object[0]) as IBackupDatabase;
try
{
int numDevices = storage.GetNumberOfDevices(storageConfig.Parameters);
string instanceName = databaseComp.GetInstanceName(databaseConfig.Parameters);
string clusterNetworkName = databaseComp.GetClusterNetworkName(databaseConfig.Parameters);
SqlPlatform sqlPlatform = VirtualBackupDeviceFactory.DiscoverSqlPlatform(instanceName,
clusterNetworkName);
//NotifyWhenReady notifyWhenReady = new NotifyWhenReady(deviceName, isBackup);))
using (IVirtualDeviceSet deviceSet = VirtualBackupDeviceFactory.NewVirtualDeviceSet(sqlPlatform))
{
using (SqlThread sql = new SqlThread())
{
bool sqlStarted = false;
bool sqlFinished = false;
ParallelExecutionException exceptions = new ParallelExecutionException();
try
{
IStreamNotification streamNotification = new InternalStreamNotification(updateNotifier);
long estimatedTotalBytes;
List <string> deviceNames = sql.PreConnect(clusterNetworkName, instanceName, deviceSetName, numDevices, databaseComp, databaseConfig.Parameters, isBackup, updateNotifier, out estimatedTotalBytes);
using (DisposableList <Stream> fileStreams = new DisposableList <Stream>(isBackup ? storage.GetBackupWriter(storageConfig.Parameters) : storage.GetRestoreReader(storageConfig.Parameters, out estimatedTotalBytes)))
using (DisposableList <Stream> topOfPilelines = new DisposableList <Stream>(CreatePipeline(pipelineConfig, fileStreams, isBackup, streamNotification, estimatedTotalBytes)))
{
VirtualDeviceSetConfig config = new VirtualDeviceSetConfig();
config.Features = FeatureSet.PipeLike;
config.DeviceCount = (uint)topOfPilelines.Count;
deviceSet.CreateEx(instanceName, deviceSetName, config);
sql.BeginExecute();
sqlStarted = true;
deviceSet.GetConfiguration(TimeSpan.FromMinutes(1));
List <IVirtualDevice> devices = new List <IVirtualDevice>();
foreach (string devName in deviceNames)
{
devices.Add(deviceSet.OpenDevice(devName));
}
using (DisposableList <DeviceThread> threads = new DisposableList <DeviceThread>(devices.Count))
{
for (int i = 0; i < devices.Count; i++)
{
DeviceThread dThread = new DeviceThread();
threads.Add(dThread);
dThread.Initialize(isBackup, topOfPilelines[i], devices[i], deviceSet);
}
foreach (DeviceThread dThread in threads)
{
dThread.BeginCopy();
}
updateNotifier.OnStart();
//Console.WriteLine(string.Format("{0} started", isBackup ? "Backup" : "Restore"));
Exception sqlE = sql.EndExecute();
sqlFinished = true;
if (sqlE != null)
{
exceptions.Exceptions.Add(sqlE);
}
foreach (DeviceThread dThread in threads)
{
Exception devE = dThread.EndCopy();
if (devE != null)
{
exceptions.Exceptions.Add(devE);
}
}
}
}
}
catch (Exception e)
{
exceptions.Exceptions.Add(e);
}
finally
{
if (sqlStarted && !sqlFinished)
{
Exception sqlE = sql.EndExecute();
sqlFinished = true;
if (sqlE != null)
{
exceptions.Exceptions.Add(sqlE);
}
}
}
if (exceptions.HasExceptions)
{
throw exceptions;
}
}
}
}
catch
{
storage.CleanupOnAbort();
throw;
}
}
public static void Restore(ConfigPair storageConfig, List <ConfigPair> pipelineConfig, ConfigPair databaseConfig, IUpdateNotification updateNotifier)
{
BackupOrRestore(false, storageConfig, databaseConfig, pipelineConfig, updateNotifier);
}
private static void BackupOrRestore(int commandType, ConfigPair storageConfig, ConfigPair databaseConfig, List <ConfigPair> pipelineConfig, IUpdateNotification updateNotifier, out List <string> returndevices)
{
var isBackup = commandType == 1;
var deviceSetName = Guid.NewGuid().ToString();
var constructorInfo = storageConfig.TransformationType.GetConstructor(new Type[0]);
if (constructorInfo == null)
{
throw new ArgumentException("Unable to transformation type");
}
var storage = constructorInfo.Invoke(new object[0]) as IBackupStorage;
var constructor = databaseConfig.TransformationType.GetConstructor(new Type[0]);
if (constructor == null)
{
throw new ArgumentException("Unable to transformation type");
}
var databaseComp = constructor.Invoke(new object[0]) as IBackupDatabase;
try
{
if (storage == null)
{
returndevices = null;
return;
}
var numDevices = storage.GetNumberOfDevices(storageConfig.Parameters);
//get instance name e.g. myserver\myinstance
if (databaseComp == null)
{
returndevices = null;
return;
}
var instanceName = databaseComp.GetInstanceName(databaseConfig.Parameters);
//get the cluster networking name usually diffrent than the currently running node name
var clusterNetworkName = databaseComp.GetClusterNetworkName(databaseConfig.Parameters);
//get bit version of our platform x86,x64 or IA64
var sqlPlatform = databaseConfig.Parameters.ContainsKey("user")
? VirtualBackupDeviceFactory.DiscoverSqlPlatform(instanceName, clusterNetworkName,
databaseConfig.Parameters["user"][0],
databaseConfig.Parameters.ContainsKey("password")
? databaseConfig.Parameters["password"][0]
: null)
: VirtualBackupDeviceFactory.DiscoverSqlPlatform(instanceName, clusterNetworkName, null,
null);
//NotifyWhenReady notifyWhenReady = new NotifyWhenReady(deviceName, isBackup);))
using (var deviceSet = VirtualBackupDeviceFactory.NewVirtualDeviceSet(sqlPlatform))
{
//Since it is possible to have multiple backup/restore targets we encapsulate each call to a device in its own thread
using (var sql = new SqlThread())
{
var sqlStarted = false;
var sqlFinished = false;
returndevices = new List <string>();
var exceptions = new ParallelExecutionException();
try
{
IStreamNotification streamNotification =
new InternalStreamNotification(updateNotifier);
long estimatedTotalBytes;
var deviceNames = sql.PreConnect(clusterNetworkName, instanceName, deviceSetName,
numDevices, databaseComp, databaseConfig.Parameters, commandType, updateNotifier,
out estimatedTotalBytes);
using (
var fileStreams =
new DisposableList <Stream>(isBackup
? storage.GetBackupWriter(storageConfig.Parameters, estimatedTotalBytes)
: storage.GetRestoreReader(storageConfig.Parameters,
out estimatedTotalBytes)))
using (
var topOfPilelines =
new DisposableList <Stream>(CreatePipeline(pipelineConfig, fileStreams,
isBackup, streamNotification, estimatedTotalBytes)))
{
ReturnByteScale(estimatedTotalBytes, isBackup);
var config = new VirtualDeviceSetConfig
{
Features = FeatureSet.PipeLike,
DeviceCount = (uint)topOfPilelines.Count
};
deviceSet.CreateEx(instanceName, deviceSetName, config);
sql.BeginExecute();
sqlStarted = true;
deviceSet.GetConfiguration(TimeSpan.FromMinutes(360));
var devices = deviceNames.Select(deviceSet.OpenDevice).ToList();
returndevices = deviceNames;
using (var threads = new DisposableList <DeviceThread>(devices.Count))
{
for (var i = 0; i < devices.Count; i++)
{
var dThread = new DeviceThread();
threads.Add(dThread);
dThread.Initialize(isBackup, topOfPilelines[i], devices[i], deviceSet);
}
foreach (var dThread in threads)
{
dThread.BeginCopy();
}
updateNotifier.OnStart();
var sqlE = sql.EndExecute();
sqlFinished = true;
if (sqlE != null)
{
exceptions.Exceptions.Add(sqlE);
}
foreach (var devE in threads.Select(dThread => dThread.EndCopy()).Where(devE => devE != null))
{
exceptions.Exceptions.Add(devE);
}
}
}
}
catch (Exception e)
{
exceptions.Exceptions.Add(e);
}
finally
{
if (exceptions.HasExceptions)
{
throw exceptions;
}
if (sqlStarted && !sqlFinished)
{
var sqlE = sql.EndExecute();
if (sqlE != null)
{
exceptions.Exceptions.Add(sqlE);
}
}
}
}
}
}
catch
{
if (storage != null)
{
storage.CleanupOnAbort();
}
throw;
}
}
public static void HeaderOnly(ConfigPair storageConfig, List <ConfigPair> pipelineConfig, ConfigPair databaseConfig, IUpdateNotification updateNotifier)
{
List <string> returndevices;
BackupOrRestore(4, storageConfig, databaseConfig, pipelineConfig, updateNotifier, out returndevices);
}
