private static int GetValidDirectionsIndex(DMLibDataType dataType)
{
switch (dataType)
{
case DMLibDataType.Stream:
return(0);
case DMLibDataType.URI:
return(1);
case DMLibDataType.Local:
return(2);
case DMLibDataType.CloudFile:
return(3);
case DMLibDataType.BlockBlob:
return(4);
case DMLibDataType.PageBlob:
return(5);
case DMLibDataType.AppendBlob:
return(6);
default:
throw new ArgumentException(string.Format("Invalid data type {0}", dataType), "dataType");;
}
}
private static IEnumerable <DMLibTransferDirection> EnumerateAllDirections(bool[][] syncDirections, bool[][] asyncDirections)
{
for (int sourceIndex = 0; sourceIndex < DataTypes.Length; ++sourceIndex)
{
for (int destIndex = 0; destIndex < DataTypes.Length; ++destIndex)
{
DMLibDataType sourceDataType = DataTypes[sourceIndex];
DMLibDataType destDataType = DataTypes[destIndex];
if (syncDirections[sourceIndex][destIndex])
{
yield return(new DMLibTransferDirection()
{
SourceType = sourceDataType,
DestType = destDataType,
IsAsync = false,
});
}
if (asyncDirections[sourceIndex][destIndex])
{
yield return(new DMLibTransferDirection()
{
SourceType = sourceDataType,
DestType = destDataType,
IsAsync = true,
});
}
}
}
}
public void UploadVariedSizeObject_Nonseekable_Positive()
{
DMLibDataType[] destTypes = new DMLibDataType[]
{
DMLibDataType.BlockBlob,
DMLibDataType.AppendBlob,
DMLibDataType.PageBlob,
DMLibDataType.CloudFile
};
foreach (var destType in destTypes)
{
if (DMLibDataType.PageBlob != destType)
{
UploadFromStreamTest(destType, VariedFileSize, null, false, true);
}
else
{
List <long> variedFileSize = new List <long>();
foreach (var fileSize in VariedFileSize)
{
variedFileSize.Add((int)Math.Ceiling(((double)fileSize) / 512) * 512);
}
UploadFromStreamTest(destType, variedFileSize.ToArray(), null, false, true);
}
}
}
public DMLibDirectionFilter(string queryString = null)
{
this.IsAsync = null;
this.SourceType = DMLibDataType.Unspecified;
this.DestType = DMLibDataType.Unspecified;
this.SetProperties(queryString);
}
public DMLibDirectionFilter(string queryString = null)
{
this.CopyMethod = null;
this.SourceType = DMLibDataType.Unspecified;
this.DestType = DMLibDataType.Unspecified;
this.SetProperties(queryString);
}
public void UploadVariedSizeObject_Nonseekable_NonfixedSized_Positive()
{
DMLibDataType[] destTypes = new DMLibDataType[] { DMLibDataType.BlockBlob, DMLibDataType.AppendBlob };
foreach (var destType in destTypes)
{
UploadFromStreamTest(destType, VariedFileSize, null, false, false);
}
}
public static void WaitUntilFileCreated(string rootPath, FileNode fileNode, DataAdaptor<DMLibDataInfo> dataAdaptor, DMLibDataType dataType, int timeoutInSec = 300)
{
Func<bool> checkFileCreated = null;
if (dataType == DMLibDataType.Local)
{
string filePath = dataAdaptor.GetAddress() + fileNode.GetLocalRelativePath();
checkFileCreated = () =>
{
return File.Exists(filePath);
};
}
else if (dataType == DMLibDataType.PageBlob ||
dataType == DMLibDataType.AppendBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlob cloudBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode);
return cloudBlob.Exists(options: HelperConst.DefaultBlobOptions);
};
}
else if (dataType == DMLibDataType.BlockBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlockBlob blockBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode) as CloudBlockBlob;
try
{
return blockBlob.DownloadBlockList(BlockListingFilter.All, options: HelperConst.DefaultBlobOptions).Any();
}
catch (StorageException)
{
return false;
}
};
}
else if (dataType == DMLibDataType.CloudFile)
{
CloudFileDataAdaptor fileAdaptor = dataAdaptor as CloudFileDataAdaptor;
checkFileCreated = () =>
{
CloudFile cloudFile = fileAdaptor.GetCloudFileReference(rootPath, fileNode);
return cloudFile.Exists(options: HelperConst.DefaultFileOptions);
};
}
else
{
Test.Error("Unexpected data type: {0}", DMLibTestContext.SourceType);
}
MultiDirectionTestHelper.WaitUntil(checkFileCreated, timeoutInSec);
}
public void DownloadToMultipleStreams_Positive()
{
DownloadToMultipleStreams(DMLibDataType.BlockBlob, random.Next(200, 1025) * 1024 * 1024, (random.Next(4, 101) / 4) * 4 * 1024 * 1024);
DMLibDataType[] sourceTypes = new DMLibDataType[] { DMLibDataType.BlockBlob, DMLibDataType.AppendBlob, DMLibDataType.PageBlob, DMLibDataType.CloudFile };
foreach (var sourceType in sourceTypes)
{
DownloadToMultipleStreams(sourceType, random.Next(10, 100) * 1024 * 1024);
}
}
private static void CleanupData(SourceOrDest sourceOrDest, DMLibDataType dataType)
{
if (SourceOrDest.Source == sourceOrDest)
{
GetSourceAdaptor(dataType).Cleanup();
}
else
{
GetDestAdaptor(dataType).Cleanup();
}
}
private static DMLibDataInfo GetDataInfo(SourceOrDest sourceOrDest, DMLibDataType dataType)
{
var dataInfos = GetDataInfos(sourceOrDest);
if (!dataInfos.ContainsKey(dataType.ToString()))
{
throw new ArgumentException(
string.Format("{0} transfer data info of specified dataType doesn't exist: {1}", sourceOrDest == SourceOrDest.Source ? "Source" : "Destination", dataType.ToString()), "dataType");
}
return dataInfos[dataType.ToString()];
}
public void UploadLargeObject_Nonseekable_NonfixedSized_Positive()
{
DMLibDataType[] destTypes = new DMLibDataType[] { DMLibDataType.BlockBlob, DMLibDataType.AppendBlob };
foreach (var destType in destTypes)
{
int length4MB = 4 * 1024 * 1024;
int blockSize = (random.Next(length4MB, 100 * 1024 * 1024) / length4MB) * length4MB;
UploadFromStreamTest(destType,
random.Next(10, 200) * 1024 * 1024,
blockSize,
false,
false);
}
}
private static void PrepareData(SourceOrDest sourceOrDest, DMLibDataType dataType, DMLibDataInfo dataInfo)
{
var dataInfos = GetDataInfos(sourceOrDest);
CleanupData(sourceOrDest, dataType);
if (SourceOrDest.Source == sourceOrDest)
{
GetSourceAdaptor(dataType).GenerateData(dataInfo);
SetSourceDataInfo(dataType, dataInfo);
}
else
{
GetDestAdaptor(dataType).GenerateData(dataInfo);
SetDestDataInfo(dataType, dataInfo);
}
}
private static IEnumerable <DMLibTransferDirection> EnumerateAllDirections(bool[][] syncDirections, bool[][] serviceSyncDirections, bool[][] asyncDirections)
{
for (int sourceIndex = 0; sourceIndex < DataTypes.Length; ++sourceIndex)
{
for (int destIndex = 0; destIndex < DataTypes.Length; ++destIndex)
{
DMLibDataType sourceDataType = DataTypes[sourceIndex];
DMLibDataType destDataType = DataTypes[destIndex];
if (syncDirections[sourceIndex][destIndex])
{
yield return(new DMLibTransferDirection()
{
SourceType = sourceDataType,
DestType = destDataType,
CopyMethod = CopyMethod.SyncCopy,
});
}
if (serviceSyncDirections[sourceIndex][destIndex])
{
yield return(new DMLibTransferDirection()
{
SourceType = sourceDataType,
DestType = destDataType,
CopyMethod = CopyMethod.ServiceSideSyncCopy,
});
}
if (asyncDirections[sourceIndex][destIndex])
{
yield return(new DMLibTransferDirection()
{
SourceType = sourceDataType,
DestType = destDataType,
CopyMethod = CopyMethod.ServiceSideAsyncCopy,
});
}
}
}
}
public static IEnumerable <DMLibDataType> Extract(this DMLibDataType type)
{
DMLibDataType[] dataTypesToExtract =
{
DMLibDataType.Stream,
DMLibDataType.URI,
DMLibDataType.Local,
DMLibDataType.CloudFile,
DMLibDataType.BlockBlob,
DMLibDataType.PageBlob,
DMLibDataType.AppendBlob,
};
foreach (var dataType in dataTypesToExtract)
{
if (type.HasFlag(dataType))
{
yield return(dataType);
}
}
}
private void DownloadToMultipleStreams(DMLibDataType sourceType, long fileSize, int blockSize = 0)
{
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(sourceType);
DataAdaptor <DMLibDataInfo> destAdaptor1 = GetDestAdaptor(DMLibDataType.Stream);
Test.Info("SourceType: {0}, filesize {1}, blocksize {2}", sourceType, fileSize, blockSize);
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
long fileSizeInByte = fileSize;
sourceDataInfo.RootNode.AddFileNode(new FileNode(FileName)
{
SizeInByte = fileSizeInByte,
});
sourceAdaptor.GenerateData(sourceDataInfo);
destAdaptor1.CreateIfNotExists();
List <TransferItem> downloadItems = new List <TransferItem>();
FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(FileName);
int streamCount = random.Next(2, 6);
for (int i = 0; i < streamCount; ++i)
{
downloadItems.Add(new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(string.Empty, fileNode),
DestObject = (destAdaptor1 as LocalDataAdaptor).GetTransferObject(string.Empty, $"{FileName}_{i}"),
SourceType = sourceType,
DestType = DMLibDataType.Stream,
IsServiceCopy = false
});
}
var options = new TestExecutionOptions <DMLibDataInfo>()
{
DisableDestinationFetch = true,
};
if (0 != blockSize)
{
options.BlockSize = blockSize;
}
// Execution
var result = this.RunTransferItems(
downloadItems,
options);
// Verify all files are transfered successfully
Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs.");
DMLibDataInfo destDataInfo = destAdaptor1.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
var toBeValidateFileNode = destFileNode.Clone(FileName);
FileNode sourceFileNode = sourceDataInfo.RootNode.GetFileNode(FileName);
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, toBeValidateFileNode), "Verify transfer result.");
}
sourceAdaptor.Cleanup();
destAdaptor1.Cleanup();
}
private static string GetTransferString(DMLibDataType sourceType, DMLibDataType destType, CopyMethod copyMethod)
{
return(sourceType.ToString() + destType.ToString() + copyMethod.ToString());
}
private void UploadFromStreamTest(DMLibDataType destType, long[] variedSourceLength, int?blockSize, bool seekable, bool fixedSized)
{
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(DMLibDataType.Stream);
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destType);
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
foreach (long fileSizeInByte in variedSourceLength)
{
sourceDataInfo.RootNode.AddFileNode(new FileNode($"{FileName}_{fileSizeInByte}")
{
SizeInByte = fileSizeInByte,
});
}
sourceAdaptor.GenerateData(sourceDataInfo);
destAdaptor.CreateIfNotExists();
List <TransferItem> uploadItems = new List <TransferItem>();
foreach (long fileSizeInByte in variedSourceLength)
{
string fileName = $"{FileName}_{fileSizeInByte}";
FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName);
uploadItems.Add(new TransferItem()
{
SourceObject = new DMLibTestStream(sourceAdaptor.GetTransferObject(string.Empty, fileNode) as FileStream, seekable, fixedSized),
DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode),
SourceType = DMLibDataType.Stream,
DestType = destType,
IsServiceCopy = false
});
}
// Execution
var result = this.RunTransferItems(
uploadItems,
new TestExecutionOptions <DMLibDataInfo>()
{
DisableDestinationFetch = true,
BlockSize = blockSize.HasValue ? blockSize.Value : 4 * 1024 * 1024
});
if (!fixedSized && (destType == DMLibDataType.PageBlob || destType == DMLibDataType.CloudFile))
{
Test.Assert(result.Exceptions.Count == 1 && result.Exceptions[0].Message.Contains("Source must be fixed size"), "Verify error is expected.");
}
else
{
// Verify all files are transfered successfully
Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs.");
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
FileNode sourceFileNode = sourceDataInfo.RootNode.GetFileNode(destFileNode.Name);
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
sourceAdaptor.Cleanup();
destAdaptor.Cleanup();
}
private void UploadFromStreamTest(DMLibDataType destType, long sourceLength, int?blockSize, bool seekable, bool fixedSized)
{
this.UploadFromStreamTest(destType, new long[] { sourceLength }, blockSize, seekable, fixedSized);
}
private static string GetTransferString(DMLibDataType sourceType, DMLibDataType destType, bool isAsync)
{
return sourceType.ToString() + destType.ToString() + (isAsync ? "async" : "");
}
protected static void PrepareDestData(DMLibDataType dataType, DMLibDataInfo dataInfo)
{
PrepareData(SourceOrDest.Dest, dataType, dataInfo);
}
public static void WaitUntilFileCreated(string rootPath, FileNode fileNode, DataAdaptor <DMLibDataInfo> dataAdaptor, DMLibDataType dataType, int timeoutInSec = 300)
{
Func <bool> checkFileCreated = null;
if (dataType == DMLibDataType.Local)
{
string filePath = dataAdaptor.GetAddress() + fileNode.GetLocalRelativePath();
checkFileCreated = () =>
{
return(File.Exists(filePath));
};
}
else if (dataType == DMLibDataType.PageBlob ||
dataType == DMLibDataType.AppendBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlob cloudBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode);
return(cloudBlob.Exists(options: HelperConst.DefaultBlobOptions));
};
}
else if (dataType == DMLibDataType.BlockBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlockBlob blockBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode) as CloudBlockBlob;
try
{
return(blockBlob.DownloadBlockList(BlockListingFilter.All, options: HelperConst.DefaultBlobOptions).Any());
}
#if EXPECT_INTERNAL_WRAPPEDSTORAGEEXCEPTION
catch (Exception e) when(e is StorageException || (e is AggregateException && e.InnerException is StorageException))
#else
catch (StorageException)
#endif
{
return(false);
}
};
}
else if (dataType == DMLibDataType.CloudFile)
{
CloudFileDataAdaptor fileAdaptor = dataAdaptor as CloudFileDataAdaptor;
checkFileCreated = () =>
{
CloudFile cloudFile = fileAdaptor.GetCloudFileReference(rootPath, fileNode);
return(cloudFile.Exists(options: HelperConst.DefaultFileOptions));
};
}
else
{
Test.Error("Unexpected data type: {0}", DMLibTestContext.SourceType);
}
MultiDirectionTestHelper.WaitUntil(checkFileCreated, timeoutInSec);
}
protected static void PrepareDestData(DMLibDataType dataType, DMLibDataInfo dataInfo)
{
PrepareData(SourceOrDest.Dest, dataType, dataInfo);
}
private static void CleanupData(SourceOrDest sourceOrDest, DMLibDataType dataType)
{
if (SourceOrDest.Source == sourceOrDest)
{
GetSourceAdaptor(dataType).Cleanup();
}
else
{
GetDestAdaptor(dataType).Cleanup();
}
}
private static int GetValidDirectionsIndex(DMLibDataType dataType)
{
switch (dataType)
{
case DMLibDataType.Stream:
return 0;
case DMLibDataType.URI:
return 1;
case DMLibDataType.Local:
return 2;
case DMLibDataType.CloudFile:
return 3;
case DMLibDataType.BlockBlob:
return 4;
case DMLibDataType.PageBlob:
return 5;
case DMLibDataType.AppendBlob:
return 6;
default:
throw new ArgumentException(string.Format("Invalid data type {0}", dataType), "dataType");;
}
}
protected static void SetDestDataInfo(DMLibDataType dataType, DMLibDataInfo dataInfo)
{
destDataInfos[dataType.ToString()] = dataInfo;
}
protected static void CleanupDestData(DMLibDataType dataType)
{
CleanupData(SourceOrDest.Dest, dataType);
}
protected static void SetSourceDataInfo(DMLibDataType dataType, DMLibDataInfo dataInfo)
{
sourceDataInfos[dataType.ToString()] = dataInfo;
}
private static string GetTransferString(DMLibDataType sourceType, DMLibDataType destType, bool isAsync)
{
return(sourceType.ToString() + destType.ToString() + (isAsync ? "async" : ""));
}
