public virtual void DownloadFileToBlob(BlobDownloadParameters parameters)
{
if (parameters == null || string.IsNullOrWhiteSpace(parameters.SasUri.ToString()))
{
throw new ArgumentNullException(Resources.DownloadCredentialsNull);
}
var rli = new PSRunLogInfo(parameters.SasUri);
StorageCredentials sc = new StorageCredentials(rli.SasToken);
StorageUri suri = new StorageUri(new Uri("https://" + parameters.SasUri.Host));
CloudBlobClient sourceClient = new CloudBlobClient(suri, sc);
CloudBlobContainer sascontainer = sourceClient.GetContainerReference(rli.Container);
CloudBlobDirectory sourceDirectory = sascontainer.GetDirectoryReference(rli.Directory);
// TODO: Remove IfDef
#if NETSTANDARD
var bloblist = sourceDirectory.ListBlobsSegmentedAsync(true, BlobListingDetails.None, null, null, null, null).Result.Results;
#else
var bloblist = sourceDirectory.ListBlobs(true);
#endif
var downloadFolderPath = parameters.Directory.Insert(parameters.Directory.Length, @"\");
foreach (var blob in bloblist)
{
ICloudBlob destBlob = blob as ICloudBlob;
int length = destBlob.Name.Split('/').Length;
string blobFileName = destBlob.Name.Split('/')[length - 1];
// the folder structure of run logs changed from flat listing to nesting under time directory
var blobFolderPath = String.Empty;
if (destBlob.Name.Length > blobFileName.Length)
{
blobFolderPath = destBlob.Name.Substring(0, destBlob.Name.Length - blobFileName.Length - 1);
if (!string.IsNullOrEmpty(rli.Directory))
{
blobFolderPath = blobFolderPath.Remove(0, rli.Directory.Length - 1).Trim('/');
}
}
if (!Directory.Exists(downloadFolderPath + blobFolderPath))
{
Directory.CreateDirectory(downloadFolderPath + blobFolderPath);
}
// adding _log suffix to differentiate between files and folders of the same name. Azure blob storage only knows about blob files. We could use nested folder structure
// as part of the blob file name and thus it is possible to have a file and folder of the same name in the same location which is not acceptable for Windows file system
string fileToDownload = destBlob.Name.Remove(0, rli.Directory.Length);
// TODO: Remove IfDef
#if NETSTANDARD
Task.Run(() => destBlob.DownloadToFileAsync(downloadFolderPath + fileToDownload + "_log", FileMode.Create)).Wait();
#else
destBlob.DownloadToFile(downloadFolderPath + fileToDownload + "_log", FileMode.Create);
#endif
}
}
public void CanSaveDataFactoryRunLog()
{
Uri sharedAccessSignature =
new Uri(@"https://fakeaccount.blob.core.windows.net/sascontainer?sv=2012-02-12");
PSRunLogInfo runLogInfo = new PSRunLogInfo(sharedAccessSignature);
Assert.Equal("?sv=2012-02-12", runLogInfo.SasToken);
Assert.Equal(@"https://fakeaccount.blob.core.windows.net/sascontainer?sv=2012-02-12", runLogInfo.SasUri);
Assert.Equal("fakeaccount", runLogInfo.StorageAccountName);
Assert.Equal("sascontainer", runLogInfo.Container);
dataFactoriesClientMock.Setup(
f =>
f.CreatePSDataFactory(
It.Is <CreatePSDataFactoryParameters>(
parameters =>
parameters.ResourceGroupName == ResourceGroupName &&
parameters.DataFactoryName == DataFactoryName &&
parameters.Location == Location)))
.CallBase()
.Verifiable();
// Arrange
this.dataFactoriesClientMock.Setup(
f =>
f.GetDataSliceRunLogsSharedAccessSignature(ResourceGroupName, DataFactoryName, this._dataSliceRunId))
.Returns(runLogInfo);
this.dataFactoriesClientMock.Setup(
f =>
f.DownloadFileToBlob(
It.Is <BlobDownloadParameters>(
parameters =>
parameters.Credentials == new StorageCredentials(runLogInfo.SasToken) &&
parameters.SasUri == new Uri(runLogInfo.SasUri) &&
parameters.Directory == @"c:\")));
// Action
this._cmdlet.ExecuteCmdlet();
// Assert
this.dataFactoriesClientMock.Verify(
f =>
f.GetDataSliceRunLogsSharedAccessSignature(ResourceGroupName, DataFactoryName, this._dataSliceRunId),
Times.Once());
this.commandRuntimeMock.Verify(f => f.WriteObject(runLogInfo), Times.Once());
}
public override void ExecuteCmdlet()
{
if (ParameterSetName == ByFactoryObject)
{
if (DataFactory == null)
{
throw new PSArgumentNullException(string.Format(CultureInfo.InvariantCulture, Resources.DataFactoryArgumentInvalid));
}
DataFactoryName = DataFactory.DataFactoryName;
ResourceGroupName = DataFactory.ResourceGroupName;
}
PSRunLogInfo runLog =
DataFactoryClient.GetDataSliceRunLogsSharedAccessSignature(
ResourceGroupName, DataFactoryName, Id);
if (DownloadLogs.IsPresent)
{
string directory = string.IsNullOrWhiteSpace(Output)
? Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments)
: Output;
if (!HaveWriteAccess(directory))
{
throw new IOException(string.Format(CultureInfo.InvariantCulture, Resources.NoWriteAccessToDirectory, directory));
}
try
{
DataFactoryClient.DownloadFileToBlob(new BlobDownloadParameters()
{
Directory = directory,
SasUri = new Uri(runLog.SasUri),
Credentials = new StorageCredentials(runLog.SasToken)
});
}
catch
{
throw new Exception(string.Format(CultureInfo.InvariantCulture, Resources.DownloadFailed, directory));
}
WriteWarning(string.Format(CultureInfo.InvariantCulture, Resources.DownloadLogCompleted, directory));
}
WriteObject(runLog);
}