protected override async Task PerformIO() { int numOfChunks = Math.Max(FileContent.Length / MAX_FILE_SIZE_BYTES + 1, 2); //Add 1 so that we are under the max file size limit, since an integer will truncate remainders. int chunkSize = FileContent.Length / numOfChunks; string sessionId = null; int index = 0; for (int i = 1; i <= numOfChunks; i++) { if (DoCancel) { throw new Exception("Operation cancelled by user"); } bool lastChunk = i == numOfChunks; int curChunkSize = chunkSize; if (lastChunk) { curChunkSize = FileContent.Length - index; } using (MemoryStream memStream = new MemoryStream(FileContent, index, curChunkSize)) { if (i == 1) { UploadSessionStartResult result = await _client.Files.UploadSessionStartAsync(false, memStream); sessionId = result.SessionId; } else { UploadSessionCursor cursor = new UploadSessionCursor(sessionId, (ulong)index); if (lastChunk) { //Always forcing Dropbox to overwrite any conflicting files. //Otherwise a Dropbox.Api.Files.UploadSessionFinishError.Path error will be returned by Dropbox if there is a "path/conflict/file/..." await _client.Files.UploadSessionFinishAsync(cursor , new CommitInfo(ODFileUtils.CombinePaths(Folder, FileName, '/'), WriteMode.Overwrite.Instance) , memStream); } else { await _client.Files.UploadSessionAppendV2Async(cursor, false, memStream); } } index += curChunkSize; } OnProgress((double)(chunkSize * i) / (double)1024 / (double)1024, "?currentVal MB of ?maxVal MB uploaded", (double)FileContent.Length / (double)1024 / (double)1024, ""); } }
/// <summary> /// Uploads a big file in chunk. The is very helpful for uploading large file in slow network condition /// and also enable capability to track upload progerss. /// </summary> /// <param name="client">The Dropbox client.</param> /// <param name="sourceFile">Byte array to upload.</param> /// <param name="location">Location on dropbox.</param> /// <returns></returns> public static async Task ChunkUpload(DropboxClient client, string sourceFile, string location) { Console.WriteLine($"{DateTime.Now} Chunk upload file..."); const int chunkSize = 1024 * 1024 * 10; using (FileStream stream = File.OpenRead(sourceFile)) { int numChunks = (int)Math.Ceiling((double)stream.Length / chunkSize); byte[] buffer = new byte[chunkSize]; string sessionId = null; for (int idx = 0; idx < numChunks; idx++) { Console.WriteLine($"{DateTime.Now} Start uploading chunk {idx}"); int byteRead = stream.Read(buffer, 0, chunkSize); using (MemoryStream memStream = new MemoryStream(buffer, 0, byteRead)) { if (idx == 0) { UploadSessionStartResult result = await client.Files.UploadSessionStartAsync(body : memStream); sessionId = result.SessionId; } else { UploadSessionCursor cursor = new UploadSessionCursor(sessionId, (ulong)(chunkSize * idx)); if (idx == numChunks - 1) { await client.Files.UploadSessionFinishAsync(cursor, new CommitInfo(location, WriteMode.Overwrite.Instance), memStream); } else { await client.Files.UploadSessionAppendV2Async(cursor, body : memStream); } } } } } }
static async Task Main(string[] args) { try { var token = args[0]; var localDirectory = Path.GetFullPath(args[1]); if (!IsEndingWithSeparator(localDirectory)) { localDirectory += Path.DirectorySeparatorChar; } var dropboxDirectory = args[2]; if (!IsEndingWithSeparator(dropboxDirectory)) { dropboxDirectory += Path.AltDirectorySeparatorChar; } string password = args[3]; var newFiles = new HashSet <string>( Directory.GetFiles(localDirectory, "*", SearchOption.AllDirectories) .Select(f => f.Substring(localDirectory.Length)), StringComparer.OrdinalIgnoreCase); var filesToDelete = new HashSet <string>(); using (var dropbox = new DropboxClient(token)) { try { await dropbox.Files.CreateFolderV2Async(dropboxDirectory.TrimEnd('/')); } catch { } var existingFiles = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase); var existingFolders = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase); existingFolders.Add(""); for (var list = await dropbox.Files.ListFolderAsync(dropboxDirectory.TrimEnd('/'), true, limit: 2000); list != null; list = list.HasMore ? await dropbox.Files.ListFolderContinueAsync(list.Cursor) : null) { foreach (var entry in list.Entries) { if (!entry.IsFile) { if (entry.IsFolder) { existingFolders.Add(entry.AsFolder.PathLower); } continue; } existingFiles.Add(entry.PathLower); var relativePath = entry.PathLower.Substring(dropboxDirectory.Length); if (!relativePath.EndsWith(".zip")) { continue; } var withoutZip = relativePath.Substring(0, relativePath.Length - 4).Replace("/", Path.DirectorySeparatorChar + ""); if (newFiles.Contains(withoutZip)) { var info = new FileInfo(Path.Combine(localDirectory, withoutZip)); if ((info.LastWriteTimeUtc - entry.AsFile.ClientModified).TotalSeconds < 1f) { newFiles.Remove(withoutZip); } } else { filesToDelete.Add(entry.PathLower); } } } await DeleteFilesBatchAsync(); ulong deletingAccumulatedSize = 0; async Task DeleteFilesBatchAsync() { if (filesToDelete.Count > 0) { Console.WriteLine($"Deleting files: \n{string.Join("\n", filesToDelete)}"); var j = await dropbox.Files.DeleteBatchAsync(filesToDelete.Select(x => new DeleteArg(x))); if (j.IsAsyncJobId) { for (DeleteBatchJobStatus r = await dropbox.Files.DeleteBatchCheckAsync(j.AsAsyncJobId.Value); r.IsInProgress; r = await dropbox.Files.DeleteBatchCheckAsync(j.AsAsyncJobId.Value)) { Thread.Sleep(5000); } } filesToDelete.Clear(); deletingAccumulatedSize = 0; } } if (newFiles.Count > 0) { Console.WriteLine($"Uploading files: {newFiles.Count}"); ZipStrings.UseUnicode = true; ZipStrings.CodePage = 65001; var entryFactory = new ZipEntryFactory(); byte[] msBuffer = new byte[1000 * 1000 * 150]; int bufferSize = 1000 * 1000 * 140; using var reader = new AsyncMultiFileReader(bufferSize, (f, t) => new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, true)); var newFilesList = newFiles.ToList(); for (int i = 0; i < newFilesList.Count; i++) { var relativePath = newFilesList[i]; Console.Write($" {relativePath}"); string fullPath = Path.Combine(localDirectory, relativePath); reader.NextFile = (fullPath, null); reader.OpenNextFile(); if (i < newFilesList.Count - 1) { reader.NextFile = (Path.Combine(localDirectory, newFilesList[i + 1]), null); } var info = new FileInfo(fullPath); var clientModifiedAt = info.LastWriteTimeUtc; using (var zipWriterUnderlyingStream = new CopyStream()) { var bufferStream = new MemoryStream(msBuffer); bufferStream.SetLength(0); UploadSessionStartResult session = null; long offset = 0; using (var zipWriter = new ZipOutputStream(zipWriterUnderlyingStream, bufferSize) { IsStreamOwner = false, Password = password, UseZip64 = UseZip64.On }) { try { zipWriterUnderlyingStream.CopyTo = bufferStream; zipWriter.SetLevel(0); var entry = entryFactory.MakeFileEntry(fullPath, '/' + Path.GetFileName(relativePath), true); entry.AESKeySize = 256; zipWriter.PutNextEntry(entry); int read; while ((read = reader.ReadNextBlock()) > 0) { Console.Write($"\r {relativePath} {offset / (double) info.Length * 100:F0}%"); zipWriter.Write(reader.CurrentBuffer, 0, read); zipWriter.Flush(); bufferStream.Position = 0; var length = bufferStream.Length; if (session == null) { session = await dropbox.Files.UploadSessionStartAsync(new UploadSessionStartArg(), bufferStream); } else { await dropbox.Files.UploadSessionAppendV2Async(new UploadSessionCursor(session.SessionId, (ulong)offset), false, bufferStream); } offset += length; zipWriterUnderlyingStream.CopyTo = bufferStream = new MemoryStream(msBuffer); bufferStream.SetLength(0); } Console.Write($"\r {relativePath} 100%"); Console.WriteLine(); zipWriter.CloseEntry(); zipWriter.Finish(); zipWriter.Close(); } catch { // disposing ZipOutputStream causes writing to bufferStream if (!bufferStream.CanRead && !bufferStream.CanWrite) { zipWriterUnderlyingStream.CopyTo = bufferStream = new MemoryStream(msBuffer); } throw; } } bufferStream.Position = 0; var commitInfo = new CommitInfo(Path.Combine(dropboxDirectory, relativePath.Replace("\\", "/")) + ".zip", WriteMode.Overwrite.Instance, false, clientModifiedAt); if (session == null) { await dropbox.Files.UploadAsync(commitInfo, bufferStream); } else { await dropbox.Files.UploadSessionFinishAsync(new UploadSessionCursor(session.SessionId, (ulong)offset), commitInfo, bufferStream); } } } } Console.WriteLine("Recycling deleted files for endless storage"); const ulong deletingBatchSize = 1024UL * 1024 * 1024 * 32; for (var list = await dropbox.Files.ListFolderAsync(dropboxDirectory.TrimEnd('/'), true, limit: 2000, includeDeleted: true); list != null; list = list.HasMore ? await dropbox.Files.ListFolderContinueAsync(list.Cursor) : null) { foreach (var entry in list.Entries) { if (!entry.IsDeleted || existingFiles.Contains(entry.PathLower)) { continue; } var parentFolder = entry.PathLower; int lastSlash = parentFolder.LastIndexOf('/'); if (lastSlash == -1) { continue; } parentFolder = parentFolder.Substring(0, lastSlash); if (!existingFolders.Contains(parentFolder)) { continue; } ListRevisionsResult rev; try { rev = await dropbox.Files.ListRevisionsAsync(entry.AsDeleted.PathLower, ListRevisionsMode.Path.Instance, 1); } catch { // get revisions doesn't work for folders but no way to check if it's a folder beforehand continue; } if (!(DateTime.UtcNow - rev.ServerDeleted >= TimeSpan.FromDays(15)) || (DateTime.UtcNow - rev.ServerDeleted > TimeSpan.FromDays(29))) { // don't need to restore too young // can't restore too old continue; } Console.WriteLine("Restoring " + entry.PathDisplay); var restored = await dropbox.Files.RestoreAsync(entry.PathLower, rev.Entries.First().Rev); if (restored.AsFile.Size >= deletingBatchSize && filesToDelete.Count == 0) { Console.WriteLine("Deleting " + entry.PathDisplay); await dropbox.Files.DeleteV2Async(restored.PathLower, restored.Rev); } else { // warning: rev not included, concurrent modification changes may be lost filesToDelete.Add(restored.PathLower); deletingAccumulatedSize += restored.Size; if (deletingAccumulatedSize >= deletingBatchSize) { await DeleteFilesBatchAsync(); } } } } await DeleteFilesBatchAsync(); } Console.WriteLine("All done"); } catch (Exception e) { // redirecting error to normal output Console.WriteLine(e); throw; } }