protected AsyncUploaderBase(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int? expirationDays = null)
: base(client, uploadSpecificationRequest, file, expirationDays)
{
Config = config ?? new FileUploaderConfig();
HashProvider = MD5HashProviderFactory.GetHashProvider().CreateHash();
}
private static long SaveTransmissionToFile(Transmission transmission, IPlatformFile file)
{
using (Stream stream = file.Open())
{
transmission.Save(stream);
return(stream.Length);
}
}
private static Transmission LoadFromTransmissionFile(IPlatformFile file, out long fileSize)
{
ChangeFileExtension(file, TemporaryFileExtension);
Transmission transmission = LoadFromTemporaryFile(file, out fileSize);
file.Delete();
return(transmission);
}
public FilePartSource(IPlatformFile file, IMD5HashProvider hashProvider)
{
fileHash = hashProvider;
stream = file.OpenRead();
fileLength = file.Length;
streamPosition = 0;
partCount = 0;
HasMore = true;
}
public ScalingFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int? expirationDays = null)
: base(client, uploadSpecificationRequest, file, config, expirationDays)
{
UploadSpecificationRequest.Raw = true;
var partConfig = config != null ? config.PartConfig : new FilePartConfig();
partUploader = new ScalingPartUploader(partConfig, Config.NumberOfThreads,
requestMessage => Task.Factory.StartNew(() => ExecuteChunkUploadMessage(requestMessage)),
OnProgress);
}
protected UploaderBase(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, int? expirationDays)
{
Client = client;
UploadSpecificationRequest = uploadSpecificationRequest;
File = file;
ExpirationDays = expirationDays;
Progress = new TransferProgress(uploadSpecificationRequest.FileSize, null, Guid.NewGuid().ToString());
}
public void CreatesFileWithSpecifiedName()
{
var folder = new PlatformFolder(this.storageFolder);
string fileName = GetUniqueFileName();
IPlatformFile file = folder.CreateFile(fileName);
Assert.AreEqual(fileName, file.Name);
}
protected UploaderBase(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, int?expirationDays)
{
Client = client;
UploadSpecificationRequest = uploadSpecificationRequest;
File = file;
ExpirationDays = expirationDays;
Progress = new TransferProgress(uploadSpecificationRequest.FileSize, null, Guid.NewGuid().ToString());
}
public AsyncScalingFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int? expirationDays = null)
: base(client, uploadSpecificationRequest, file, config, expirationDays)
{
UploadSpecificationRequest.Raw = true;
var chunkConfig = config != null ? config.PartConfig : new FilePartConfig();
partUploader = new ScalingPartUploader(chunkConfig, Config.NumberOfThreads,
ExecuteChunkUploadMessage,
OnProgress);
}
public FileViewModel(IPlatformFile file)
{
Id = file.Id;
Name = file.Name;
Version = file.Version;
data = file;
Password = file.Password;
CommentsCount = file.CommentsCount;
FolderId = (int)file.ParentId.GetValueOrDefault();
Versions = new ObservableCollection<VersionViewModel>(file.Versions.Select(a => new VersionViewModel(a)));
}
public void CreatesPhysicalFileInFileSystem()
{
var folder = new PlatformFolder(this.storageFolder);
string fileName = GetUniqueFileName();
IPlatformFile file = folder.CreateFile(fileName);
var storageFile = FileSystemTest.GetPlatformFile(fileName, this.storageFolder);
Assert.IsNotNull(storageFile);
}
public virtual Transmission Dequeue()
{
if (this.folder == null)
{
return(null);
}
this.EnsureSizeIsCalculated();
string lastInaccessibleFileName = null;
while (true)
{
IPlatformFile file = null;
try
{
file = this.GetOldestTransmissionFileOrNull();
if (file == null)
{
return(null); // because there are no more transmission files
}
long fileSize;
Transmission transmission = LoadFromTransmissionFile(file, out fileSize);
if (transmission != null)
{
Interlocked.Add(ref this.size, -fileSize);
return(transmission);
}
}
catch (UnauthorizedAccessException)
{
if (file == null)
{
return(null); // because the process does not have permission to access the folder
}
if (lastInaccessibleFileName != file.Name)
{
lastInaccessibleFileName = file.Name;
continue; // because another thread is loading this file right now
}
throw; // because the process does not have permission to modify this file
}
catch (IOException)
{
continue; // because another thread already loaded this file
}
}
}
public Task Upload(IPlatformFile file, IMD5HashProvider hashProvider, string chunkUploadUrl)
{
return(Task.Factory.StartNew(() =>
{
var workers = Dispatch(new FilePartSource(file, hashProvider), chunkUploadUrl).ToArray();
Task.WaitAll(workers);
var results = workers.Select(task => task.Result);
if (!results.All(partUploadResult => partUploadResult.IsSuccess))
{
throw results.Select(result => result.PartUploadException).FirstOrDefault(ex => ex != null) ?? new UploadException("FilePart upload failed", -1);
}
}));
}
private static Transmission LoadFromTemporaryFile(IPlatformFile file, out long fileSize)
{
using (Stream stream = file.Open())
{
try
{
fileSize = stream.Length;
return(TransmissionExtensions.Load(stream));
}
catch (FormatException exp)
{
fileSize = 0;
TelemetryChannelEventSource.Log.IncorrectFileFormatWarning(exp.Message);
return(null);
}
}
}
public virtual bool Enqueue(Func <Transmission> transmissionGetter)
{
if (this.folder == null)
{
return(false);
}
this.EnsureSizeIsCalculated();
if (this.size < this.Capacity)
{
var transmission = transmissionGetter();
if (transmission == null)
{
return(false);
}
try
{
IPlatformFile temporaryFile = this.CreateTemporaryFile();
long temporaryFileSize = SaveTransmissionToFile(transmission, temporaryFile);
ChangeFileExtension(temporaryFile, TransmissionFileExtension);
Interlocked.Add(ref this.size, temporaryFileSize);
TelemetryChannelEventSource.Log.TransmissionSavedToStorage(transmission.Id);
return(true);
}
catch (UnauthorizedAccessException e)
{
// Expected because the process may have lost permission to access the folder or the files in it.
TelemetryChannelEventSource.Log.UnauthorizedAccessExceptionOnTransmissionSaveWarning(transmission.Id, e.Message);
}
catch (Exception exp)
{
TelemetryChannelEventSource.Log.TransmissionFailedToStoreWarning(transmission.Id, exp.ToString());
throw;
}
}
else
{
TelemetryChannelEventSource.Log.StorageEnqueueNoCapacityWarning(this.size, this.Capacity);
}
return(false);
}
private static Transmission LoadFromTransmissionFile(IPlatformFile file, out long fileSize)
{
fileSize = 0;
Transmission transmission = null;
if (file.Exists)
{
// The ingestion service rejects anything older than 2 days.
if (file.DateCreated > DateTimeOffset.Now.AddDays(-2))
{
ChangeFileExtension(file, TemporaryFileExtension);
transmission = LoadFromTemporaryFile(file, out fileSize);
}
else
{
TelemetryChannelEventSource.Log.TransmissionStorageFileExpired(file.Name, file.DateCreated.ToString(CultureInfo.InvariantCulture));
}
file.Delete();
}
return(transmission);
}
public virtual Transmission Dequeue()
{
if (this.folder == null || this.flushAsyncInProcessCounter > 0)
{
return(null);
}
this.EnsureSizeIsCalculated();
while (true)
{
IPlatformFile file = null;
try
{
if (this.flushAsyncInProcessCounter == 0)
{
file = this.GetOldestTransmissionFileOrNull();
if (file == null)
{
return(null); // Because there are no more transmission files.
}
long fileSize;
Transmission transmission = LoadFromTransmissionFile(file, out fileSize);
if (transmission != null)
{
Interlocked.Add(ref this.size, -fileSize);
return(transmission);
}
}
}
catch (UnauthorizedAccessException uae)
{
if (file == null)
{
TelemetryChannelEventSource.Log.TransmissionStorageDequeueUnauthorizedAccessException(this.folder.Name, uae.ToString());
return(null); // Because the process does not have permission to access the folder.
}
string name = file.Name;
if (this.badFiles.TryAdd(name, null))
{
TelemetryChannelEventSource.Log.TransmissionStorageInaccessibleFile(name);
}
else
{
// The same file has been inaccessible more than once because the process does not have permission to modify this file.
TelemetryChannelEventSource.Log.TransmissionStorageUnexpectedRetryOfBadFile(name);
}
Thread.Sleep(this.random.Next(1, 100)); // Sleep for random time of 1 to 100 milliseconds to try to avoid future timing conflicts.
}
catch (IOException ioe)
{
// This exception can happen when one thread runs out of files to process and reloads the list while another
// thread is still processing a file and has not deleted it yet thus allowing it to get in the list again.
TelemetryChannelEventSource.Log.TransmissionStorageDequeueIOError(file.Name, ioe.ToString());
Thread.Sleep(this.random.Next(1, 100)); // Sleep for random time of 1 to 100 milliseconds to try to avoid future timing conflicts.
continue; // It may be because another thread already loaded this file, we don't know yet.
}
}
}
private static long SaveTransmissionToFile(Transmission transmission, IPlatformFile file)
{
using (Stream stream = file.Open())
{
transmission.Save(stream);
return stream.Length;
}
}
private static void ChangeFileExtension(IPlatformFile file, string extension)
{
string transmissionFileName = GetUniqueFileName(extension);
file.Rename(transmissionFileName);
}
private static void ChangeFileExtension(IPlatformFile file, string extension)
{
string transmissionFileName = GetUniqueFileName(extension);
file.Rename(transmissionFileName);
}
public FileInfoArgs(IPlatformFile file)
{
_file = file;
}
/// <summary>
///
/// </summary>
/// <param name="uploadSpecificationRequest"></param>
/// <param name="file"></param>
/// <param name="config"></param>
/// <param name="expirationDays">Will only be obeyed by ShareFile apps</param>
/// <returns></returns>
public SyncUploaderBase GetFileUploader(UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null)
{
this.PreprocessUploadSpecRequest(uploadSpecificationRequest);
switch (uploadSpecificationRequest.Method)
{
case UploadMethod.Standard:
return(new StandardFileUploader(this, uploadSpecificationRequest, file, config, expirationDays));
break;
case UploadMethod.Threaded:
return(new ScalingFileUploader(this, uploadSpecificationRequest, file, config, expirationDays));
break;
}
throw new NotSupportedException(uploadSpecificationRequest.Method + " is not supported.");
}
public AsyncStandardFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int? expirationDays = null)
: base(client, uploadSpecificationRequest, file, config, expirationDays)
{
UploadSpecificationRequest.Raw = true;
}
public AsyncScalingFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null)
: base(client, uploadSpecificationRequest, file, config, expirationDays)
{
UploadSpecificationRequest.Raw = true;
var chunkConfig = config != null ? config.PartConfig : new FilePartConfig();
partUploader = new ScalingPartUploader(chunkConfig, Config.NumberOfThreads,
ExecuteChunkUploadMessage,
OnProgress);
}
private static Transmission LoadFromTemporaryFile(IPlatformFile file, out long fileSize)
{
using (Stream stream = file.Open())
{
try
{
fileSize = stream.Length;
return TransmissionExtensions.Load(stream);
}
catch (FormatException exp)
{
fileSize = 0;
TelemetryChannelEventSource.Log.IncorrectFileFormatWarning(exp.Message);
return null;
}
}
}
public AsyncStandardFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null)
: base(client, uploadSpecificationRequest, file, config, expirationDays)
{
UploadSpecificationRequest.Raw = false;
}
private static void ChangeFileExtension(IPlatformFile file, string extension)
{
string transmissionFileName = Path.ChangeExtension(file.Name, extension);
file.Rename(transmissionFileName);
}
private static Transmission LoadFromTransmissionFile(IPlatformFile file, out long fileSize)
{
ChangeFileExtension(file, TemporaryFileExtension);
Transmission transmission = LoadFromTemporaryFile(file, out fileSize);
file.Delete();
return transmission;
}
protected AsyncUploaderBase(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null)
: base(client, uploadSpecificationRequest, file, expirationDays)
{
Config = config ?? new FileUploaderConfig();
HashProvider = MD5HashProviderFactory.GetHashProvider().CreateHash();
}
public ScalingFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null)
: base(client, uploadSpecificationRequest, file, config, expirationDays)
{
UploadSpecificationRequest.Raw = true;
var partConfig = config != null ? config.PartConfig : new FilePartConfig();
partUploader = new ScalingPartUploader(partConfig, Config.NumberOfThreads,
requestMessage => Task.Factory.StartNew(() => ExecuteChunkUploadMessage(requestMessage)),
OnProgress);
}
