public void StartDownloading( //intput ConcurrentQueue<int> blobIds, RateLimiter rateLimiter, //output BlockingCollection<Tuple<int, MemoryStream>> blobContents, BlockingCollection<long> downloadedBytes, //cancellation CancellationTokenSource cts, CancellationToken ct) { try { this.oracleConn.Open(); } catch (Exception) { cts.Cancel(); throw; } int blobId; while (blobIds.TryDequeue(out blobId)) { ct.ThrowIfCancellationRequested(); try { MemoryStream ms = new MemoryStream(); using (var cmd = (OracleCommand)oracleConn.CreateStoreCommand( @"SELECT CONTENTS FROM CAA_DOC.DOCLIB_DOCUMENTS WHERE {0}", new DbClause("DOC_ID = {0}", blobId))) using (var reader = cmd.ExecuteReader()) { reader.Read(); using (var blob = reader.GetOracleBlob(reader.GetOrdinal("CONTENTS"))) { blob.CopyTo(ms); ms.Position = 0; } } downloadedBytes.Add(ms.Length); rateLimiter.Increment(ms.Length); blobContents.Add(Tuple.Create(blobId, ms)); } catch (Exception) { cts.Cancel(); throw; } } }
public void StartUploading( //intput BlockingCollection<Tuple<int, MemoryStream>> blobContents, RateLimiter rateLimiter, //output ConcurrentDictionary<int, string> blobIdsToFileKeys, BlockingCollection<long> uploadedBytes, //cancellation CancellationTokenSource cts, CancellationToken ct) { try { this.sqlConn.Open(); } catch (Exception) { cts.Cancel(); throw; } foreach (var blobContent in blobContents.GetConsumingEnumerable()) { ct.ThrowIfCancellationRequested(); var blobId = blobContent.Item1; var content = blobContent.Item2; try { using (var blobWriter = new BlobWriter(sqlConn)) { long length; using (var stream = blobWriter.OpenStream()) using (content) { length = content.Length; content.CopyTo(stream); } if (!blobIdsToFileKeys.TryAdd(blobId, blobWriter.GetBlobKey().ToString())) { throw new Exception("blobId already present in dictionary"); } rateLimiter.Decrement(length); uploadedBytes.Add(length); } } catch (Exception) { cts.Cancel(); throw; } } }
public Dictionary<int, string> migrateBlobs() { this.oracleConn.Open(); Stopwatch timer = new Stopwatch(); timer.Start(); var ids = this.oracleConn.CreateStoreCommand(@"SELECT DOC_ID FROM CAA_DOC.DOCLIB_DOCUMENTS") .Materialize(r => r.Field<int>("DOC_ID")); Dictionary<int, string> blobIdsToFileKeys; bool isFakeBlobMigration; if (!bool.TryParse(ConfigurationManager.AppSettings["FakeBlobMigration"], out isFakeBlobMigration)) { isFakeBlobMigration = false; } if (isFakeBlobMigration) { blobIdsToFileKeys = ids.ToDictionary(id => id, id => DUMMY_FILE_KEY); } else { CancellationTokenSource cts = new CancellationTokenSource(); CancellationToken ct = cts.Token; ConcurrentDictionary<int, string> blobIdsToFileKeysConcurrent = new ConcurrentDictionary<int, string>(); BlockingCollection<Tuple<int, MemoryStream>> blobContents = new BlockingCollection<Tuple<int, MemoryStream>>(); ConcurrentQueue<int> blobIds = new ConcurrentQueue<int>(ids); BlockingCollection<long> downloadedBytes = new BlockingCollection<long>(); BlockingCollection<long> uploadedBytes = new BlockingCollection<long>(); int maxUploadQueueSizeInMB; if (!int.TryParse(ConfigurationManager.AppSettings["MaxUploadQueueSizeInMB"], out maxUploadQueueSizeInMB)) { maxUploadQueueSizeInMB = 8 * 1024;//8GB default } RateLimiter rateLimiter = new RateLimiter(0, maxUploadQueueSizeInMB * 1024 * 1024L, ct); Task.WaitAll( Task.WhenAll( Utils.RunParallel("ParallelDownloads", ct, () => this.downloaderFactory().Value, (downloader) => { using (downloader) { downloader.StartDownloading(blobIds, rateLimiter, blobContents, downloadedBytes, cts, ct); } }) .ContinueWith((t) => { blobContents.CompleteAdding(); if (t.IsFaulted) { throw t.Exception; } }), Utils.RunParallel("ParallelUploads", ct, () => this.uploaderFactory().Value, (uploader) => { using (uploader) { uploader.StartUploading(blobContents, rateLimiter, blobIdsToFileKeysConcurrent, uploadedBytes, cts, ct); } })) .ContinueWith((t) => { downloadedBytes.CompleteAdding(); uploadedBytes.CompleteAdding(); if (t.IsFaulted) { throw t.Exception; } }), Task.Run(() => PrintInfo(Console.CursorTop, "Downloaded", downloadedBytes, rateLimiter)), Task.Run(() => PrintInfo(Console.CursorTop + 1, "Uploaded", uploadedBytes, rateLimiter))); blobIdsToFileKeys = blobIdsToFileKeysConcurrent.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); } timer.Stop(); Console.WriteLine("Blob migration time - {0}", timer.Elapsed.TotalMinutes); return blobIdsToFileKeys; }
private void PrintInfo(int cursorTop, string task, BlockingCollection<long> bytes, RateLimiter rateLimitier) { int count = 0; double totalSpeedKBPerSec = 0; double periodSpeedKBPerSec = 0; Stopwatch totalTimer = new Stopwatch(); totalTimer.Start(); long totalBytes = 0; Stopwatch periodTimer = new Stopwatch(); periodTimer.Start(); long periodBytes = 0; long periodMs = 5000; long maxSize = 0; foreach (var b in bytes.GetConsumingEnumerable()) { count++; totalBytes += b; periodBytes += b; if (periodTimer.ElapsedMilliseconds > periodMs) { periodSpeedKBPerSec = (((double)periodBytes) / 1024) / ((((double)periodTimer.ElapsedMilliseconds) + 1) / 1000); periodBytes = 0; periodTimer.Restart(); } totalSpeedKBPerSec = (((double)totalBytes) / 1024) / ((((double)totalTimer.ElapsedMilliseconds) + 1) / 1000); maxSize = Math.Max(maxSize, rateLimitier.CurrentSize); Console.SetCursorPosition(0, cursorTop); Console.Write( string.Format("{0,-10} {1,6} | Avg: {2,7} KB/s | Current {3,7} KB/s {4}", task, count, Math.Round(totalSpeedKBPerSec, 1).ToString("0.0"), Math.Round(periodSpeedKBPerSec, 1).ToString("0.0"), maxSize) .PadRight(80)); } totalTimer.Stop(); periodTimer.Stop(); }