private async Task <bool> DoPut(FileEntryItem item, bool updatedHash = false) { // If this is not already encrypted, do it now item.Encrypt(m_options); updatedHash |= item.UpdateHashAndSize(m_options); if (updatedHash && item.TrackedInDb) { await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash); } if (m_options.Dryrun) { Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length)); item.DeleteLocalFile(); return(true); } await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash })); await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size); var begin = DateTime.Now; if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers) { using (var fs = System.IO.File.OpenRead(item.LocalFilename)) using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg))) ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs); } else { m_backend.Put(item.RemoteFilename, item.LocalFilename); } var duration = DateTime.Now - begin; Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds))); if (item.TrackedInDb) { await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash); } await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size); if (m_options.ListVerifyUploads) { var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); if (f == null) { throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename)); } else if (f.Size != item.Size && f.Size >= 0) { throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size)); } } item.DeleteLocalFile(); await m_database.CommitTransactionAsync("CommitAfterUpload"); return(true); }
private async Task DoPut(FileEntryItem item, IBackend backend, CancellationToken cancelToken) { if (cancelToken.IsCancellationRequested) { return; } if (item.TrackedInDb) { await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash); } if (m_options.Dryrun) { Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length)); item.DeleteLocalFile(); return; } await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash })); await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size, updateProgress : false); m_progressUpdater.StartFileProgress(item.RemoteFilename, item.Size); var begin = DateTime.Now; if (!m_options.DisableStreamingTransfers && backend is IStreamingBackend streamingBackend) { // A download throttle speed is not given to the ThrottledStream as we are only uploading data here using (var fs = File.OpenRead(item.LocalFilename)) using (var ts = new ThrottledStream(fs, m_initialUploadThrottleSpeed, 0)) using (var pgs = new ProgressReportingStream(ts, pg => HandleProgress(ts, pg, item.RemoteFilename))) await streamingBackend.PutAsync(item.RemoteFilename, pgs, cancelToken).ConfigureAwait(false); } else { await backend.PutAsync(item.RemoteFilename, item.LocalFilename, cancelToken).ConfigureAwait(false); } var duration = DateTime.Now - begin; m_progressUpdater.EndFileProgress(item.RemoteFilename); Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds))); if (item.TrackedInDb) { await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash); } await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size); if (m_options.ListVerifyUploads) { var f = backend.List().FirstOrDefault(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)); if (f == null) { throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename)); } else if (f.Size != item.Size && f.Size >= 0) { throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size)); } } item.DeleteLocalFile(); await m_database.CommitTransactionAsync("CommitAfterUpload"); }
private void DoPut(FileEntryItem item) { if (m_encryption != null) lock(m_encryptionLock) item.Encrypt(m_encryption, m_statwriter); if (item.UpdateHashAndSize(m_options) && !item.NotTrackedInDb) m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash); if (item.Indexfile != null && !item.IndexfileUpdated) { item.Indexfile.Item1.FinishVolume(item.Hash, item.Size); item.Indexfile.Item1.Close(); item.IndexfileUpdated = true; } m_db.LogDbOperation("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash })); m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size); if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers) { using (var fs = System.IO.File.OpenRead(item.LocalFilename)) using (var ts = new ThrottledStream(fs, m_options.MaxDownloadPrSecond, m_options.MaxUploadPrSecond)) using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs); } else m_backend.Put(item.RemoteFilename, item.LocalFilename); if (!item.NotTrackedInDb) m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash); m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size); if (m_options.ListVerifyUploads) { var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault(); if (f == null) throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", f.Name)); else if (f.Size != item.Size && f.Size >= 0) throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size)); } item.DeleteLocalFile(m_statwriter); }