public void AddFileNode(FileNode fileNode)
{
fileNode.Parent = this;
this.fileNodeMap.Add(fileNode.Name, fileNode);
}
public abstract object GetTransferObject(string rootPath, FileNode fileNode, StorageCredentials credentials = null);
public void TestMetadataOverwrite()
{
Dictionary <string, string> metadata = new Dictionary <string, string>();
metadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
metadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
Test.Info("Metadata is =====================");
foreach (var keyValue in metadata)
{
Test.Info("name:{0} value:{1}", keyValue.Key, keyValue.Value);
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
FileNode fileNode = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = DMLibTestBase.FileSizeInKB * 1024L,
Metadata = metadata
};
sourceDataInfo.RootNode.AddFileNode(fileNode);
Dictionary <string, string> destMetadata = new Dictionary <string, string>();
destMetadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
destMetadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
Test.Info("Destination metadata is =====================");
foreach (var keyValue in destMetadata)
{
Test.Info("name:{0} value:{1}", keyValue.Key, keyValue.Value);
}
DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty);
fileNode = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = DMLibTestBase.FileSizeInKB * 1024L,
Metadata = destMetadata
};
destDataInfo.RootNode.AddFileNode(fileNode);
var option = new TestExecutionOptions <DMLibDataInfo>()
{
DestTransferDataInfo = destDataInfo
};
SingleTransferContext transferContext = new SingleTransferContext();
transferContext.ShouldOverwriteCallbackAsync = async(source, dest) =>
{
return(true);
};
option.TransferItemModifier = (inputFileNode, transferItem) =>
{
transferItem.TransferContext = transferContext;
};
var result = this.ExecuteTestCase(sourceDataInfo, option);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
public static bool Equals(DirNode dirNodeA, DirNode dirNodeB)
{
// The same node
if (dirNodeA == dirNodeB)
{
return(true);
}
// Empty node equals to null
if ((dirNodeA == null || dirNodeA.IsEmpty) &&
(dirNodeB == null || dirNodeB.IsEmpty))
{
return(true);
}
// Compare two nodes
if (null != dirNodeA && null != dirNodeB)
{
if (dirNodeA.FileNodeCount != dirNodeB.FileNodeCount ||
dirNodeA.NonEmptyDirNodeCount != dirNodeB.NonEmptyDirNodeCount)
{
return(false);
}
foreach (FileNode fileNodeA in dirNodeA.FileNodes)
{
FileNode fileNodeB = dirNodeB.GetFileNode(fileNodeA.Name);
FileNode fileNodeAA = fileNodeA;
if (null == fileNodeB)
{
fileNodeB = dirNodeB.GetFileNode(DMLibTestHelper.EscapeInvalidCharacters(fileNodeA.Name));
if (null != fileNodeB)
{
fileNodeAA = fileNodeA.Clone(DMLibTestHelper.EscapeInvalidCharacters(fileNodeA.Name));
}
}
if (!DMLibDataHelper.Equals(fileNodeAA, fileNodeB))
{
return(false);
}
}
foreach (DirNode subDirNodeA in dirNodeA.DirNodes)
{
Test.Info("Verifying subfolder: {0} ", subDirNodeA.Name);
DirNode subDirNodeB = dirNodeB.GetDirNode(subDirNodeA.Name);
if (null == subDirNodeB)
{
subDirNodeB = dirNodeB.GetDirNode(DMLibTestHelper.EscapeInvalidCharacters(subDirNodeA.Name));
}
if (!DMLibDataHelper.Equals(subDirNodeA, subDirNodeB))
{
return(false);
}
}
return(true);
}
return(false);
}
public void ReleaseLease(string rootPath, FileNode fileNode, string leaseId)
{
var blob = this.GetCloudBlobReference(rootPath, fileNode);
blob.ReleaseLease(AccessCondition.GenerateLeaseCondition(leaseId), options: HelperConst.DefaultBlobOptions);
}
public override object GetTransferObject(string rootPath, FileNode fileNode, StorageCredentials credentials = null)
{
return(this.GetCloudFileReference(rootPath, fileNode, credentials));
}
public string LeaseBlob(string rootPath, FileNode fileNode, TimeSpan?leaseTime)
{
var blob = this.GetCloudBlobReference(rootPath, fileNode);
return(blob.AcquireLease(leaseTime, null, options: HelperConst.DefaultBlobOptions));
}
private void GenerateFile(FileNode fileNode, CloudBlob cloudBlob, List <FileNode> snapshotList)
{
this.CheckFileNode(fileNode);
if ((StorageBlob.BlobType.PageBlob == cloudBlob.BlobType) && (fileNode.SizeInByte % 512 != 0))
{
throw new InvalidOperationException(string.Format("Can only generate page blob which size is multiple of 512bytes. Expected size is {0}", fileNode.SizeInByte));
}
string tempFileName = Guid.NewGuid().ToString();
string localFilePath = Path.Combine(this.TempFolder, tempFileName);
DMLibDataHelper.CreateLocalFile(fileNode, localFilePath);
BlobRequestOptions storeMD5Options = new BlobRequestOptions()
{
RetryPolicy = DMLibTestConstants.DefaultRetryPolicy,
StoreBlobContentMD5 = true,
MaximumExecutionTime = DMLibTestConstants.DefaultExecutionTimeOut
};
if (fileNode.BlockSize.HasValue && cloudBlob is CloudBlockBlob)
{
((CloudBlockBlob)cloudBlob).StreamWriteSizeInBytes = fileNode.BlockSize.Value;
}
cloudBlob.UploadFromFile(localFilePath, null, storeMD5Options);
if (null != fileNode.MD5 ||
null != fileNode.ContentType ||
null != fileNode.CacheControl ||
null != fileNode.ContentDisposition ||
null != fileNode.ContentEncoding ||
null != fileNode.ContentLanguage)
{
cloudBlob.Properties.ContentMD5 = fileNode.MD5;
cloudBlob.Properties.ContentType = fileNode.ContentType;
cloudBlob.Properties.CacheControl = fileNode.CacheControl;
cloudBlob.Properties.ContentDisposition = fileNode.ContentDisposition;
cloudBlob.Properties.ContentEncoding = fileNode.ContentEncoding;
cloudBlob.Properties.ContentLanguage = fileNode.ContentLanguage;
cloudBlob.SetProperties(options: HelperConst.DefaultBlobOptions);
}
if (null != fileNode.Metadata && fileNode.Metadata.Count > 0)
{
cloudBlob.Metadata.Clear();
foreach (var metaData in fileNode.Metadata)
{
cloudBlob.Metadata.Add(metaData);
}
cloudBlob.SetMetadata(options: HelperConst.DefaultBlobOptions);
}
cloudBlob.FetchAttributes(options: HelperConst.DefaultBlobOptions);
this.BuildFileNode(cloudBlob, fileNode);
for (int i = 0; i < fileNode.SnapshotsCount; ++i)
{
CloudBlob snapshot = cloudBlob.Snapshot();
snapshotList.Add(this.BuildSnapshotFileNode(snapshot, fileNode.Name));
}
}
public static void WaitUntilFileCreated(string rootPath, FileNode fileNode, DataAdaptor <DMLibDataInfo> dataAdaptor, DMLibDataType dataType, int timeoutInSec = 300)
{
Func <bool> checkFileCreated = null;
if (dataType == DMLibDataType.Local)
{
string filePath = dataAdaptor.GetAddress() + fileNode.GetLocalRelativePath();
checkFileCreated = () =>
{
return(File.Exists(filePath));
};
}
else if (dataType == DMLibDataType.PageBlob ||
dataType == DMLibDataType.AppendBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlob cloudBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode);
return(cloudBlob.Exists(options: HelperConst.DefaultBlobOptions));
};
}
else if (dataType == DMLibDataType.BlockBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlockBlob blockBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode) as CloudBlockBlob;
try
{
return(blockBlob.DownloadBlockList(BlockListingFilter.All, options: HelperConst.DefaultBlobOptions).Any());
}
#if EXPECT_INTERNAL_WRAPPEDSTORAGEEXCEPTION
catch (Exception e) when(e is StorageException || (e is AggregateException && e.InnerException is StorageException))
#else
catch (StorageException)
#endif
{
return(false);
}
};
}
else if (dataType == DMLibDataType.CloudFile)
{
CloudFileDataAdaptor fileAdaptor = dataAdaptor as CloudFileDataAdaptor;
checkFileCreated = () =>
{
CloudFile cloudFile = fileAdaptor.GetCloudFileReference(rootPath, fileNode);
return(cloudFile.Exists(options: HelperConst.DefaultFileOptions));
};
}
else
{
Test.Error("Unexpected data type: {0}", DMLibTestContext.SourceType);
}
MultiDirectionTestHelper.WaitUntil(checkFileCreated, timeoutInSec);
}
public void ResumeInAllDirections()
{
List <TransferItem> allItems = AllTransferDirectionTest.GetTransformItemsForAllDirections(resume: true);
int fileCount = expectedFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
progressChecker.DataTransferred.WaitOne();
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach (var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
transferContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
TransferEventChecker eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
List <TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return(itemToResume);
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
foreach (DMLibDataType destDataType in DataTypes)
{
DataAdaptor <DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
FileNode sourceFileNode = expectedFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
Test.Assert(result.Exceptions.Count == 0, "Verify no error happens. Actual: {0}", result.Exceptions.Count);
}
}
private void GenerateFile(FileNode fileNode, CloudFile cloudFile, string parentPath)
{
this.CheckFileNode(fileNode);
string tempFileName = Guid.NewGuid().ToString();
string localFilePath = Path.Combine(parentPath, tempFileName);
DMLibDataHelper.CreateLocalFile(fileNode, localFilePath);
FileRequestOptions storeMD5Options = new FileRequestOptions()
{
RetryPolicy = DMLibTestConstants.DefaultRetryPolicy,
StoreFileContentMD5 = true,
MaximumExecutionTime = DMLibTestConstants.DefaultExecutionTimeOut
};
cloudFile.UploadFromFile(localFilePath, options: storeMD5Options);
if (null != fileNode.MD5 ||
null != fileNode.ContentType ||
null != fileNode.CacheControl ||
null != fileNode.ContentDisposition ||
null != fileNode.ContentEncoding ||
null != fileNode.ContentLanguage)
{
// set user defined MD5 to cloud file
cloudFile.Properties.ContentMD5 = fileNode.MD5;
cloudFile.Properties.ContentType = fileNode.ContentType;
cloudFile.Properties.CacheControl = fileNode.CacheControl;
cloudFile.Properties.ContentDisposition = fileNode.ContentDisposition;
cloudFile.Properties.ContentEncoding = fileNode.ContentEncoding;
cloudFile.Properties.ContentLanguage = fileNode.ContentLanguage;
cloudFile.SetProperties(options: HelperConst.DefaultFileOptions);
}
if (null != fileNode.SMBAttributes)
{
cloudFile.Properties.NtfsAttributes = fileNode.SMBAttributes;
cloudFile.Properties.CreationTime = fileNode.CreationTime;
cloudFile.Properties.LastWriteTime = fileNode.LastWriteTime;
cloudFile.SetProperties(options: HelperConst.DefaultFileOptions);
}
if (null != fileNode.PortableSDDL)
{
cloudFile.FilePermission = fileNode.PortableSDDL;
cloudFile.SetProperties(options: HelperConst.DefaultFileOptions);
}
if (null != fileNode.Metadata && fileNode.Metadata.Count > 0)
{
cloudFile.Metadata.Clear();
foreach (var metaData in fileNode.Metadata)
{
cloudFile.Metadata.Add(metaData);
}
cloudFile.SetMetadata(options: HelperConst.DefaultFileOptions);
}
this.BuildFileNode(cloudFile, fileNode);
}
