protected AsyncUploaderBase(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int? expirationDays = null) : base(client, uploadSpecificationRequest, file, expirationDays) { Config = config ?? new FileUploaderConfig(); HashProvider = MD5HashProviderFactory.GetHashProvider().CreateHash(); }
private static long SaveTransmissionToFile(Transmission transmission, IPlatformFile file) { using (Stream stream = file.Open()) { transmission.Save(stream); return(stream.Length); } }
private static Transmission LoadFromTransmissionFile(IPlatformFile file, out long fileSize) { ChangeFileExtension(file, TemporaryFileExtension); Transmission transmission = LoadFromTemporaryFile(file, out fileSize); file.Delete(); return(transmission); }
public FilePartSource(IPlatformFile file, IMD5HashProvider hashProvider) { fileHash = hashProvider; stream = file.OpenRead(); fileLength = file.Length; streamPosition = 0; partCount = 0; HasMore = true; }
public ScalingFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int? expirationDays = null) : base(client, uploadSpecificationRequest, file, config, expirationDays) { UploadSpecificationRequest.Raw = true; var partConfig = config != null ? config.PartConfig : new FilePartConfig(); partUploader = new ScalingPartUploader(partConfig, Config.NumberOfThreads, requestMessage => Task.Factory.StartNew(() => ExecuteChunkUploadMessage(requestMessage)), OnProgress); }
protected UploaderBase(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, int? expirationDays) { Client = client; UploadSpecificationRequest = uploadSpecificationRequest; File = file; ExpirationDays = expirationDays; Progress = new TransferProgress(uploadSpecificationRequest.FileSize, null, Guid.NewGuid().ToString()); }
public void CreatesFileWithSpecifiedName() { var folder = new PlatformFolder(this.storageFolder); string fileName = GetUniqueFileName(); IPlatformFile file = folder.CreateFile(fileName); Assert.AreEqual(fileName, file.Name); }
protected UploaderBase(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, int?expirationDays) { Client = client; UploadSpecificationRequest = uploadSpecificationRequest; File = file; ExpirationDays = expirationDays; Progress = new TransferProgress(uploadSpecificationRequest.FileSize, null, Guid.NewGuid().ToString()); }
public AsyncScalingFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int? expirationDays = null) : base(client, uploadSpecificationRequest, file, config, expirationDays) { UploadSpecificationRequest.Raw = true; var chunkConfig = config != null ? config.PartConfig : new FilePartConfig(); partUploader = new ScalingPartUploader(chunkConfig, Config.NumberOfThreads, ExecuteChunkUploadMessage, OnProgress); }
public FileViewModel(IPlatformFile file) { Id = file.Id; Name = file.Name; Version = file.Version; data = file; Password = file.Password; CommentsCount = file.CommentsCount; FolderId = (int)file.ParentId.GetValueOrDefault(); Versions = new ObservableCollection<VersionViewModel>(file.Versions.Select(a => new VersionViewModel(a))); }
public void CreatesPhysicalFileInFileSystem() { var folder = new PlatformFolder(this.storageFolder); string fileName = GetUniqueFileName(); IPlatformFile file = folder.CreateFile(fileName); var storageFile = FileSystemTest.GetPlatformFile(fileName, this.storageFolder); Assert.IsNotNull(storageFile); }
public virtual Transmission Dequeue() { if (this.folder == null) { return(null); } this.EnsureSizeIsCalculated(); string lastInaccessibleFileName = null; while (true) { IPlatformFile file = null; try { file = this.GetOldestTransmissionFileOrNull(); if (file == null) { return(null); // because there are no more transmission files } long fileSize; Transmission transmission = LoadFromTransmissionFile(file, out fileSize); if (transmission != null) { Interlocked.Add(ref this.size, -fileSize); return(transmission); } } catch (UnauthorizedAccessException) { if (file == null) { return(null); // because the process does not have permission to access the folder } if (lastInaccessibleFileName != file.Name) { lastInaccessibleFileName = file.Name; continue; // because another thread is loading this file right now } throw; // because the process does not have permission to modify this file } catch (IOException) { continue; // because another thread already loaded this file } } }
public Task Upload(IPlatformFile file, IMD5HashProvider hashProvider, string chunkUploadUrl) { return(Task.Factory.StartNew(() => { var workers = Dispatch(new FilePartSource(file, hashProvider), chunkUploadUrl).ToArray(); Task.WaitAll(workers); var results = workers.Select(task => task.Result); if (!results.All(partUploadResult => partUploadResult.IsSuccess)) { throw results.Select(result => result.PartUploadException).FirstOrDefault(ex => ex != null) ?? new UploadException("FilePart upload failed", -1); } })); }
private static Transmission LoadFromTemporaryFile(IPlatformFile file, out long fileSize) { using (Stream stream = file.Open()) { try { fileSize = stream.Length; return(TransmissionExtensions.Load(stream)); } catch (FormatException exp) { fileSize = 0; TelemetryChannelEventSource.Log.IncorrectFileFormatWarning(exp.Message); return(null); } } }
public virtual bool Enqueue(Func <Transmission> transmissionGetter) { if (this.folder == null) { return(false); } this.EnsureSizeIsCalculated(); if (this.size < this.Capacity) { var transmission = transmissionGetter(); if (transmission == null) { return(false); } try { IPlatformFile temporaryFile = this.CreateTemporaryFile(); long temporaryFileSize = SaveTransmissionToFile(transmission, temporaryFile); ChangeFileExtension(temporaryFile, TransmissionFileExtension); Interlocked.Add(ref this.size, temporaryFileSize); TelemetryChannelEventSource.Log.TransmissionSavedToStorage(transmission.Id); return(true); } catch (UnauthorizedAccessException e) { // Expected because the process may have lost permission to access the folder or the files in it. TelemetryChannelEventSource.Log.UnauthorizedAccessExceptionOnTransmissionSaveWarning(transmission.Id, e.Message); } catch (Exception exp) { TelemetryChannelEventSource.Log.TransmissionFailedToStoreWarning(transmission.Id, exp.ToString()); throw; } } else { TelemetryChannelEventSource.Log.StorageEnqueueNoCapacityWarning(this.size, this.Capacity); } return(false); }
private static Transmission LoadFromTransmissionFile(IPlatformFile file, out long fileSize) { fileSize = 0; Transmission transmission = null; if (file.Exists) { // The ingestion service rejects anything older than 2 days. if (file.DateCreated > DateTimeOffset.Now.AddDays(-2)) { ChangeFileExtension(file, TemporaryFileExtension); transmission = LoadFromTemporaryFile(file, out fileSize); } else { TelemetryChannelEventSource.Log.TransmissionStorageFileExpired(file.Name, file.DateCreated.ToString(CultureInfo.InvariantCulture)); } file.Delete(); } return(transmission); }
public virtual Transmission Dequeue() { if (this.folder == null || this.flushAsyncInProcessCounter > 0) { return(null); } this.EnsureSizeIsCalculated(); while (true) { IPlatformFile file = null; try { if (this.flushAsyncInProcessCounter == 0) { file = this.GetOldestTransmissionFileOrNull(); if (file == null) { return(null); // Because there are no more transmission files. } long fileSize; Transmission transmission = LoadFromTransmissionFile(file, out fileSize); if (transmission != null) { Interlocked.Add(ref this.size, -fileSize); return(transmission); } } } catch (UnauthorizedAccessException uae) { if (file == null) { TelemetryChannelEventSource.Log.TransmissionStorageDequeueUnauthorizedAccessException(this.folder.Name, uae.ToString()); return(null); // Because the process does not have permission to access the folder. } string name = file.Name; if (this.badFiles.TryAdd(name, null)) { TelemetryChannelEventSource.Log.TransmissionStorageInaccessibleFile(name); } else { // The same file has been inaccessible more than once because the process does not have permission to modify this file. TelemetryChannelEventSource.Log.TransmissionStorageUnexpectedRetryOfBadFile(name); } Thread.Sleep(this.random.Next(1, 100)); // Sleep for random time of 1 to 100 milliseconds to try to avoid future timing conflicts. } catch (IOException ioe) { // This exception can happen when one thread runs out of files to process and reloads the list while another // thread is still processing a file and has not deleted it yet thus allowing it to get in the list again. TelemetryChannelEventSource.Log.TransmissionStorageDequeueIOError(file.Name, ioe.ToString()); Thread.Sleep(this.random.Next(1, 100)); // Sleep for random time of 1 to 100 milliseconds to try to avoid future timing conflicts. continue; // It may be because another thread already loaded this file, we don't know yet. } } }
private static long SaveTransmissionToFile(Transmission transmission, IPlatformFile file) { using (Stream stream = file.Open()) { transmission.Save(stream); return stream.Length; } }
private static void ChangeFileExtension(IPlatformFile file, string extension) { string transmissionFileName = GetUniqueFileName(extension); file.Rename(transmissionFileName); }
public FileInfoArgs(IPlatformFile file) { _file = file; }
/// <summary> /// /// </summary> /// <param name="uploadSpecificationRequest"></param> /// <param name="file"></param> /// <param name="config"></param> /// <param name="expirationDays">Will only be obeyed by ShareFile apps</param> /// <returns></returns> public SyncUploaderBase GetFileUploader(UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null) { this.PreprocessUploadSpecRequest(uploadSpecificationRequest); switch (uploadSpecificationRequest.Method) { case UploadMethod.Standard: return(new StandardFileUploader(this, uploadSpecificationRequest, file, config, expirationDays)); break; case UploadMethod.Threaded: return(new ScalingFileUploader(this, uploadSpecificationRequest, file, config, expirationDays)); break; } throw new NotSupportedException(uploadSpecificationRequest.Method + " is not supported."); }
public AsyncStandardFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int? expirationDays = null) : base(client, uploadSpecificationRequest, file, config, expirationDays) { UploadSpecificationRequest.Raw = true; }
public AsyncScalingFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null) : base(client, uploadSpecificationRequest, file, config, expirationDays) { UploadSpecificationRequest.Raw = true; var chunkConfig = config != null ? config.PartConfig : new FilePartConfig(); partUploader = new ScalingPartUploader(chunkConfig, Config.NumberOfThreads, ExecuteChunkUploadMessage, OnProgress); }
private static Transmission LoadFromTemporaryFile(IPlatformFile file, out long fileSize) { using (Stream stream = file.Open()) { try { fileSize = stream.Length; return TransmissionExtensions.Load(stream); } catch (FormatException exp) { fileSize = 0; TelemetryChannelEventSource.Log.IncorrectFileFormatWarning(exp.Message); return null; } } }
public AsyncStandardFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null) : base(client, uploadSpecificationRequest, file, config, expirationDays) { UploadSpecificationRequest.Raw = false; }
private static void ChangeFileExtension(IPlatformFile file, string extension) { string transmissionFileName = Path.ChangeExtension(file.Name, extension); file.Rename(transmissionFileName); }
private static Transmission LoadFromTransmissionFile(IPlatformFile file, out long fileSize) { ChangeFileExtension(file, TemporaryFileExtension); Transmission transmission = LoadFromTemporaryFile(file, out fileSize); file.Delete(); return transmission; }
protected AsyncUploaderBase(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null) : base(client, uploadSpecificationRequest, file, expirationDays) { Config = config ?? new FileUploaderConfig(); HashProvider = MD5HashProviderFactory.GetHashProvider().CreateHash(); }
public ScalingFileUploader(ShareFileClient client, UploadSpecificationRequest uploadSpecificationRequest, IPlatformFile file, FileUploaderConfig config = null, int?expirationDays = null) : base(client, uploadSpecificationRequest, file, config, expirationDays) { UploadSpecificationRequest.Raw = true; var partConfig = config != null ? config.PartConfig : new FilePartConfig(); partUploader = new ScalingPartUploader(partConfig, Config.NumberOfThreads, requestMessage => Task.Factory.StartNew(() => ExecuteChunkUploadMessage(requestMessage)), OnProgress); }