public static void AssertValidFileSmbProperties(FileSmbProperties fileSmbProperties)
{
Assert.IsNotNull(fileSmbProperties.FileAttributes);
Assert.IsNotNull(fileSmbProperties.FilePermissionKey);
Assert.IsNotNull(fileSmbProperties.FileCreatedOn);
Assert.IsNotNull(fileSmbProperties.FileLastWrittenOn);
Assert.IsNotNull(fileSmbProperties.FileChangedOn);
Assert.IsNotNull(fileSmbProperties.FileId);
Assert.IsNotNull(fileSmbProperties.ParentId);
}
internal static void AssertPropertiesEqual(FileSmbProperties left, FileSmbProperties right)
{
Assert.AreEqual(left.FileAttributes, right.FileAttributes);
Assert.AreEqual(left.FileCreatedOn, right.FileCreatedOn);
Assert.AreEqual(left.FileChangedOn, right.FileChangedOn);
Assert.AreEqual(left.FileId, right.FileId);
Assert.AreEqual(left.FileLastWrittenOn, right.FileLastWrittenOn);
Assert.AreEqual(left.FilePermissionKey, right.FilePermissionKey);
Assert.AreEqual(left.ParentId, right.ParentId);
}
public override void ExecuteCmdlet()
{
if (AsJob.IsPresent)
{
DoBeginProcessing();
}
string filePath = this.Source;
FileInfo localFile = new FileInfo(filePath);
if (!localFile.Exists)
{
throw new FileNotFoundException(string.Format(CultureInfo.CurrentCulture, Resources.SourceFileNotFound, this.Source));
}
long fileSize = localFile.Length;
// if FIPS policy is enabled, must use native MD5 for DMlib.
if (fipsEnabled)
{
if (fileSize < sizeTB)
{
CloudStorageAccount.UseV1MD5 = false;
}
else // use Track2 SDK
{
WriteWarning("The uploaded file won't have Content MD5 hash, since caculate MD5 hash fail, most possiblly caused by FIPS is enabled on this machine.");
}
}
bool isDirectory;
string[] path = NamingUtil.ValidatePath(this.Path, out isDirectory);
var cloudFileToBeUploaded =
BuildCloudFileInstanceFromPathAsync(localFile.Name, path, isDirectory).ConfigureAwait(false).GetAwaiter().GetResult();
if (ShouldProcess(cloudFileToBeUploaded.Name, "Set file content"))
{
// Step 2: Build the CloudFile object which pointed to the
// destination cloud file.
this.RunTask(async taskId =>
{
var progressRecord = new ProgressRecord(
this.OutputStream.GetProgressId(taskId),
string.Format(CultureInfo.CurrentCulture, Resources.SendAzureFileActivity, localFile.Name,
cloudFileToBeUploaded.GetFullPath(), cloudFileToBeUploaded.Share.Name),
Resources.PrepareUploadingFile);
if (fileSize <= sizeTB)
{
await DataMovementTransferHelper.DoTransfer(() =>
this.TransferManager.UploadAsync(
localFile.FullName,
cloudFileToBeUploaded,
new UploadOptions
{
PreserveSMBAttributes = context is null ? false : context.PreserveSMBAttribute.IsPresent
},
this.GetTransferContext(progressRecord, localFile.Length),
this.CmdletCancellationToken),
progressRecord,
this.OutputStream).ConfigureAwait(false);
}
else // use Track2 SDK
{
//Create File
ShareFileClient fileClient = AzureStorageFile.GetTrack2FileClient(cloudFileToBeUploaded, Channel.StorageContext);
// confirm overwrite if file exist
if (!this.Force.IsPresent &&
fileClient.Exists(this.CmdletCancellationToken) &&
!await this.OutputStream.ConfirmAsync(string.Format(CultureInfo.CurrentCulture, Resources.OverwriteConfirmation, Util.ConvertToString(cloudFileToBeUploaded))))
{
return;
}
await fileClient.CreateAsync(fileSize, cancellationToken: this.CmdletCancellationToken).ConfigureAwait(false);
//Prepare progress Handler
IProgress <long> progressHandler = new Progress <long>((finishedBytes) =>
{
if (progressRecord != null)
{
// Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
progressRecord.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize);
progressRecord.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, progressRecord.PercentComplete);
this.OutputStream.WriteProgress(progressRecord);
}
});
long blockSize = 4 * 1024 * 1024;
int maxWorkers = 4;
List <Task> runningTasks = new List <Task>();
IncrementalHash hash = null;
if (!fipsEnabled)
{
hash = IncrementalHash.CreateHash(HashAlgorithmName.MD5);
}
using (FileStream stream = File.OpenRead(localFile.FullName))
{
byte[] buffer = null;
long lastBlockSize = 0;
for (long offset = 0; offset < fileSize; offset += blockSize)
{
long currentBlockSize = offset + blockSize < fileSize ? blockSize : fileSize - offset;
// Only need to create new buffer when chunk size change
if (currentBlockSize != lastBlockSize)
{
buffer = new byte[currentBlockSize];
lastBlockSize = currentBlockSize;
}
await stream.ReadAsync(buffer: buffer, offset: 0, count: (int)currentBlockSize);
if (!fipsEnabled && hash != null)
{
hash.AppendData(buffer);
}
Task task = UploadFileRangAsync(fileClient,
new HttpRange(offset, currentBlockSize),
new MemoryStream(buffer),
progressHandler);
runningTasks.Add(task);
// Check if any of upload range tasks are still busy
if (runningTasks.Count >= maxWorkers)
{
await Task.WhenAny(runningTasks).ConfigureAwait(false);
// Clear any completed blocks from the task list
for (int i = 0; i < runningTasks.Count; i++)
{
Task runningTask = runningTasks[i];
if (!runningTask.IsCompleted)
{
continue;
}
await runningTask.ConfigureAwait(false);
runningTasks.RemoveAt(i);
i--;
}
}
}
// Wait for all upload range tasks finished
await Task.WhenAll(runningTasks).ConfigureAwait(false);
}
// Need set file properties
if ((!fipsEnabled && hash != null) || (context != null && context.PreserveSMBAttribute.IsPresent))
{
ShareFileHttpHeaders header = null;
if (!fipsEnabled && hash != null)
{
header = new ShareFileHttpHeaders();
header.ContentHash = hash.GetHashAndReset();
}
FileSmbProperties smbProperties = null;
if (context != null && context.PreserveSMBAttribute.IsPresent)
{
FileInfo sourceFileInfo = new FileInfo(localFile.FullName);
smbProperties = new FileSmbProperties();
smbProperties.FileCreatedOn = sourceFileInfo.CreationTimeUtc;
smbProperties.FileLastWrittenOn = sourceFileInfo.LastWriteTimeUtc;
smbProperties.FileAttributes = Util.LocalAttributesToAzureFileNtfsAttributes(File.GetAttributes(localFile.FullName));
}
// set file header and attributes to the file
fileClient.SetHttpHeaders(httpHeaders: header, smbProperties: smbProperties);
}
if (this.PassThru)
{
// fetch latest file properties for output
cloudFileToBeUploaded.FetchAttributes();
}
}
if (this.PassThru)
{
WriteCloudFileObject(taskId, this.Channel, cloudFileToBeUploaded);
}
});