Exemple #1
0
        private void TestStreamReadSpeed(ThrottledStreamRead readMethod)
        {
            // arrange
            var size = 1024;
            var maxBytesPerSecond = 256;                               // 256 Byte/s
            var slowExpectedTime  = (size / maxBytesPerSecond) * 1000; // 4000 Milliseconds
            var fastExpectedTime  = slowExpectedTime * 0.75;           // 3000 Milliseconds
            var randomBytes       = DummyData.GenerateRandomBytes(size);
            var buffer            = new byte[maxBytesPerSecond / 8];
            var readSize          = 1;

            using Stream stream = new ThrottledStream(new MemoryStream(randomBytes), maxBytesPerSecond);
            var stopWatcher = Stopwatch.StartNew();

            // act
            stream.Seek(0, SeekOrigin.Begin);
            while (readSize > 0)
            {
                readSize = readMethod(stream, buffer, 0, buffer.Length);
            }
            stopWatcher.Stop();

            // assert
            Assert.IsTrue(stopWatcher.ElapsedMilliseconds >= fastExpectedTime, $"actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
            Assert.IsTrue(stopWatcher.ElapsedMilliseconds <= slowExpectedTime, $"actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
        }
Exemple #2
0
        private void TestStreamReadByDynamicSpeed(ThrottledStreamRead readMethod)
        {
            // arrange
            var limitationCoefficient = 0.9;                                                                                            // 90%
            var size = 10240;                                                                                                           // 10KB
            var maxBytesPerSecond = 1024;                                                                                               // 1 KByte/s
            var expectedTime      = (size / 2 / maxBytesPerSecond + size / 2 / (maxBytesPerSecond * 2)) * 1000 * limitationCoefficient; // 90% of 10000 Milliseconds
            var randomBytes       = DummyData.GenerateRandomBytes(size);
            var buffer            = new byte[maxBytesPerSecond / 8];
            var readSize          = 1;
            var totalReadSize     = 0L;

            using ThrottledStream stream = new ThrottledStream(new MemoryStream(randomBytes), maxBytesPerSecond);
            var stopWatcher = Stopwatch.StartNew();

            // act
            stream.Seek(0, SeekOrigin.Begin);
            while (readSize > 0)
            {
                readSize       = readMethod(stream, buffer, 0, buffer.Length);
                totalReadSize += readSize;

                // increase speed (2X) after downloading half size
                if (totalReadSize > size / 2 && maxBytesPerSecond == stream.BandwidthLimit)
                {
                    stream.BandwidthLimit *= 2;
                }
            }
            stopWatcher.Stop();

            // assert
            Assert.IsTrue(stopWatcher.ElapsedMilliseconds >= expectedTime,
                          $"expected duration is: {expectedTime}ms , but actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
        }
Exemple #3
0
        private void HandleProgress(ThrottledStream ts, long pg)
        {
            if (!m_taskreader.TransferProgressAsync.WaitForTask().Result)
            {
                throw new OperationCanceledException();
            }

            // Update the throttle speeds if they have changed
            string tmp;

            m_options.RawOptions.TryGetValue("throttle-upload", out tmp);
            if (tmp != m_lastThrottleUploadValue)
            {
                ts.WriteSpeed             = m_options.MaxUploadPrSecond;
                m_lastThrottleUploadValue = tmp;
            }

            m_options.RawOptions.TryGetValue("throttle-download", out tmp);
            if (tmp != m_lastThrottleDownloadValue)
            {
                ts.ReadSpeed = m_options.MaxDownloadPrSecond;
                m_lastThrottleDownloadValue = tmp;
            }

            m_stats.UpdateBackendProgress(pg);
        }
Exemple #4
0
        /// <summary>
        /// Загрузить файл на сервер
        /// !!! добавить прогресс !!!
        /// </summary>
        /// <param name="localFilePath">Локальный путь и имя файла для загрузки.</param>
        /// <param name="remoteFileUri">Путь к файлу и имя файла на сервере.</param>
        public async Task <bool> UploadFileAsync(string localFilePath, Uri remoteFileUri)
        {
            bool result = false;

            Progress_Clear();
            Show_Message($"Загрузка файла {localFilePath}");

            if (quota.IsLimited)
            {
                FileInfo fileInfo = new FileInfo(localFilePath);
                if (quota.QuotaAvailableBytes < fileInfo.Length)
                {
                    LogController.Warn(logger, $"Недостаточно места на сервере");
                    Progress_Complete();
                    Show_Message();
                    return(false);
                }
            }

            try
            {
                using (WebDavSession session = new WebDavSession(Server, httpClientHandler, version))
                {
                    Progress <WebDavProgress> progress = new Progress <WebDavProgress>();
                    progress.ProgressChanged += Progress_ProgressChanged;
                    session.Timeout           = TimeSpan.FromDays(1);

                    Stream stream = File.OpenRead(localFilePath);
                    if (Settings.SpeedLimiter.IsLimitUpload)
                    {
                        stream = new ThrottledStream(stream, Settings.SpeedLimiter.UploadSpeedLimit * 1024);
                    }
                    Set_Stream(stream);

                    string contentType = MimeMapping.GetMimeMapping(localFilePath);
                    result = await session.UploadFileWithProgressAsync(remoteFileUri, stream, contentType, progress, CancellationToken.None);

                    if (result)
                    {
                        LogController.Info(logger, $"Файл успешно загружен: {localFilePath}");
                        Progress_Complete();
                    }
                    else
                    {
                        LogController.Warn(logger, $"Файл не загружен: {localFilePath}");
                    }

                    Close_Stream();
                    //stream.Dispose();
                }
            }
            catch (Exception e)
            {
                LogController.Error(logger, e, $"Ошибка загрузки файла: {localFilePath}");
            }

            Show_Message();

            return(result);
        }
        private void DownloadFile(FileLink fileLink, WebClient client)
        {
            Console.WriteLine("File {0} status - started.", fileLink.Name);
            var sppedInBps = _speedInKbps * 1024;

            _timer.Start();
            using (var stream = client.OpenRead(fileLink.HttpAddress))
            {
                var throttledStream = new ThrottledStream(stream, sppedInBps);

                var fileName = _outputPath + fileLink.Name;

                using (var file = File.Create(fileName))
                {
                    var buffer    = new byte[BufferSize];
                    var readCount = throttledStream.Read(buffer, 0, BufferSize);

                    while (readCount > 0)
                    {
                        file.Write(buffer, 0, readCount);
                        readCount = throttledStream.Read(buffer, 0, BufferSize);
                    }
                }
                throttledStream.Close();
            }
            _timer.Stop();
            Console.WriteLine("File {0} status - downloaded in {1} seconds.", fileLink.Name, _timer.ElapsedMilliseconds / 1000);
            _timer.Reset();
        }
        public void TestStreamIntegrity()
        {
            using (Stream tar = new ThrottledStream(new MemoryStream(), 100))
            {
                byte[] buf = DummyData.GenerateOrderedBytes(500);
                tar.Write(buf, 0, buf.Length);
                tar.Seek(0, SeekOrigin.Begin);
                byte[] buf2 = new byte[500];
                tar.Read(buf2, 0, buf2.Length);
                Assert.IsTrue(buf.SequenceEqual(buf2));
            }

            using (Stream tar = new ThrottledStream(new MemoryStream()))
            {
                byte[] buf = DummyData.GenerateOrderedBytes(4096);
                tar.Write(buf, 0, buf.Length);
                tar.Seek(0, SeekOrigin.Begin);
                byte[] buf2 = new byte[4096];
                tar.Read(buf2, 0, buf2.Length);
                Assert.IsTrue(buf.SequenceEqual(buf2));
            }

            using (Stream tar = new ThrottledStream(new MemoryStream(), 77))
            {
                byte[] buf = DummyData.GenerateOrderedBytes(247);
                tar.Write(buf, 0, buf.Length);
                tar.Seek(0, SeekOrigin.Begin);
                byte[] buf2 = new byte[247];
                tar.Read(buf2, 0, buf2.Length);
                Assert.IsTrue(buf.SequenceEqual(buf2));
            }
        }
Exemple #7
0
        public static void ThrottledStreamRead()
        {
            byte[]    sourceBuffer      = { 0x10, 0x20, 0x30, 0x40, 0x50 };
            byte[]    destinationBuffer = new byte[sourceBuffer.Length + 1];
            const int offset            = 1;
            const int bytesToRead       = 3;

            using (MemoryStream baseStream = new MemoryStream(sourceBuffer))
            {
                const int readSpeed  = 1;
                const int writeSpeed = 1;

                ThrottledStream throttledStream = new ThrottledStream(baseStream, readSpeed, writeSpeed);
                int             bytesRead       = throttledStream.Read(destinationBuffer, offset, bytesToRead);
                Assert.AreEqual(bytesToRead, bytesRead);

                for (int k = 0; k < destinationBuffer.Length; k++)
                {
                    if (offset <= k && k < offset + bytesToRead)
                    {
                        Assert.AreEqual(sourceBuffer[k - offset], destinationBuffer[k]);
                    }
                    else
                    {
                        Assert.AreEqual(default(byte), destinationBuffer[k]);
                    }
                }
            }
        }
        private Datamodels.TumblrJson RequestData(string url, string authHeaders)
        {
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                request.Method = "GET";
                request.ServicePoint.Expect100Continue = false;
                request.ContentType = "x-www-from-urlencoded";

                if (authHeaders != null)
                {
                    // add OAuth header
                    request.Headers["Authorization"] = authHeaders;
                }

                var jsserializer = new JavaScriptSerializer();
                using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
                {
                    ThrottledStream stream = new ThrottledStream(response.GetResponseStream(), (shellService.Settings.Bandwidth / shellService.Settings.ParallelImages) * 1024);
                    StreamReader    reader = new StreamReader(stream);

                    DataModels.TumblrJson data = jsserializer.Deserialize <DataModels.TumblrJson>(reader.ReadToEnd());
                    if (data.meta.status == 200)
                    {
                        return(data);
                    }
                    return(null);
                }
            }
            catch (WebException ex)
            {
                return(null);
            }
        }
Exemple #9
0
        private Response GetObject(string bucket, string key)
        {
            var s3Object = storage.GetObject(bucket, key);

            if (s3Object == null)
            {
                return(new Response {
                    StatusCode = HttpStatusCode.NotFound
                });
            }

            var stream = s3Object.Content();

            var response = new Response {
                StatusCode = HttpStatusCode.OK, ContentType = s3Object.ContentType
            };

            response.WithHeader("ETag", string.Format("\"{0}\"", s3Object.ContentMD5));
            response.WithHeader("Accept-Ranges", "bytes");
            response.Contents = x =>
            {
                var throttledStream = new ThrottledStream(stream, configuration.MaxBytesPerSecond);
                throttledStream.CopyTo(x);
            };

            return(response);
        }
Exemple #10
0
        private void TestStreamReadSpeed(ThrottledStreamRead readMethod)
        {
            // arrange
            var limitationCoefficient = 0.8;                                                   // 80%
            var size = 10240;                                                                  // 10KB
            var maxBytesPerSecond = 1024;                                                      // 1024 Byte/s
            var expectedTime      = (size / maxBytesPerSecond) * 1000 * limitationCoefficient; // 80% of 10000 Milliseconds
            var randomBytes       = DummyData.GenerateRandomBytes(size);
            var buffer            = new byte[maxBytesPerSecond / 8];
            var readSize          = 1;

            using Stream stream = new ThrottledStream(new MemoryStream(randomBytes), maxBytesPerSecond);
            var stopWatcher = Stopwatch.StartNew();

            // act
            stream.Seek(0, SeekOrigin.Begin);
            while (readSize > 0)
            {
                readSize = readMethod(stream, buffer, 0, buffer.Length);
            }
            stopWatcher.Stop();

            // assert
            Assert.IsTrue(stopWatcher.ElapsedMilliseconds >= expectedTime,
                          $"expected duration is: {expectedTime}ms , but actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
        }
Exemple #11
0
        public static void ThrottledStreamWrite()
        {
            byte[]    initialBuffer = { 0x10, 0x20, 0x30, 0x40, 0x50 };
            byte[]    source        = { 0x60, 0x70, 0x80, 0x90 };
            const int offset        = 1;
            const int bytesToWrite  = 3;

            using (MemoryStream baseStream = new MemoryStream(initialBuffer))
            {
                const int readSpeed  = 1;
                const int writeSpeed = 1;

                ThrottledStream throttledStream = new ThrottledStream(baseStream, readSpeed, writeSpeed);
                throttledStream.Write(source, offset, bytesToWrite);

                byte[] result = baseStream.ToArray();
                for (int k = 0; k < result.Length; k++)
                {
                    if (k < bytesToWrite)
                    {
                        Assert.AreEqual(source[offset + k], result[k]);
                    }
                    else
                    {
                        Assert.AreEqual(initialBuffer[k], result[k]);
                    }
                }
            }
        }
Exemple #12
0
        public void TestZeroBandwidth()
        {
            // arrange
            int maximumBytesPerSecond = 0;

            // act
            using var throttledStream = new ThrottledStream(new MemoryStream(), maximumBytesPerSecond);

            // assert
            Assert.AreEqual(long.MaxValue, throttledStream.BandwidthLimit);
        }
Exemple #13
0
        internal Peer(Stream stream, IConnection connection)
        {
            Connection                 = connection;
            SendSpeedWatcher           = new SpeedWatcher();
            ReceiveSpeedWatcher        = new SpeedWatcher();
            SendBandwidthController    = new UnlimitedBandwidthController();
            ReceiveBandwidthController = new UnlimitedBandwidthController();
            Statistics                 = new PeerStat();
            Stream = new ThrottledStream(stream, ReceiveBandwidthController, SendBandwidthController);

            Uri = new Uri("tcp://" + EndPoint.Address + ':' + EndPoint.Port);
        }
        public HttpResponseMessage GetImagingFile(DtoImageFileRequest fileRequest)
        {
            var guid          = ConfigurationManager.AppSettings["ComServerUniqueId"];
            var thisComServer = new ServiceClientComServer().GetServerByGuid(guid);

            if (thisComServer == null)
            {
                Logger.Error($"Com Server With Guid {guid} Not Found");
                return(new HttpResponseMessage(HttpStatusCode.NotFound));
            }
            var profile = new ServiceImageProfile().ReadProfile(fileRequest.profileId);

            if (profile == null)
            {
                Logger.Error($"Image Profile Not Found: {fileRequest.profileId}");
                return(new HttpResponseMessage(HttpStatusCode.NotFound));
            }

            var maxBitRate = thisComServer.ImagingMaxBps;
            var basePath   = thisComServer.LocalStoragePath;

            var fullPath = Path.Combine(basePath, "images", profile.Image.Name, $"hd{fileRequest.hdNumber}",
                                        fileRequest.fileName);

            if (File.Exists(fullPath))
            {
                HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
                try
                {
                    var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read);
                    if (maxBitRate == 0)
                    {
                        result.Content = new StreamContent(stream);
                    }
                    else
                    {
                        Stream throttledStream = new ThrottledStream(stream, maxBitRate);
                        result.Content = new StreamContent(throttledStream);
                    }

                    result.Content.Headers.ContentType                 = new MediaTypeHeaderValue("application/octet-stream");
                    result.Content.Headers.ContentDisposition          = new ContentDispositionHeaderValue("inline");
                    result.Content.Headers.ContentDisposition.FileName = fileRequest.fileName;
                    return(result);
                }
                catch (Exception ex)
                {
                    Logger.Error(ex.Message);
                }
            }

            return(new HttpResponseMessage(HttpStatusCode.NotFound));
        }
Exemple #15
0
        public void TestNegativeBandwidth()
        {
            // arrange
            int maximumBytesPerSecond = -1;

            // act
            void CreateThrottledStream()
            {
                using var throttledStream = new ThrottledStream(new MemoryStream(), maximumBytesPerSecond);
            }

            // assert
            Assert.ThrowsException <ArgumentOutOfRangeException>(CreateThrottledStream);
        }
        public void TestStreamWrite()
        {
            using (Stream tar = new ThrottledStream(new MemoryStream(), 256))
            {
                tar.Seek(0, SeekOrigin.Begin);
                byte[] buf   = DummyData.GenerateRandomBytes(1024);
                int    start = Environment.TickCount;

                tar.Write(buf, 0, buf.Length);


                int elapsed = Environment.TickCount - start;
                Assert.GreaterOrEqual(elapsed, 4000);
            }
        }
Exemple #17
0
        private async Task <SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string[] pathParts, SyncTarget target, SyncOptions options, IProgress <double> progress, CancellationToken cancellationToken)
        {
            _logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, string.Join("/", pathParts));
            using (var fileStream = _fileSystem.GetFileStream(inputPath, FileMode.Open, FileAccess.Read, FileShare.Read, true))
            {
                Stream stream = fileStream;

                if (options.UploadSpeedLimitBytes > 0 && provider is IRemoteSyncProvider)
                {
                    stream = new ThrottledStream(stream, options.UploadSpeedLimitBytes);
                }

                return(await provider.SendFile(stream, pathParts, target, progress, cancellationToken).ConfigureAwait(false));
            }
        }
Exemple #18
0
 /// <summary>
 /// Throttles for the specified buffer size in bytes.
 /// </summary>
 /// <param name="maximumBytesPerSecond">The maximum bytes per second that can be transferred through the base stream. Specify zero to disable this feature.</param>
 public void LimitTransferSpeed(long maximumBytesPerSecond)
 {
     if (DataStream == null)
     {
         throw new ArgumentNullException("DataStream");
     }
     if (DataStream is ThrottledStream)
     {
         ((ThrottledStream)DataStream).MaximumBytesPerSecond = maximumBytesPerSecond;
     }
     else
     {
         DataStream = new ThrottledStream(DataStream, maximumBytesPerSecond);
     }
 }
 public void UpdateFileProgress(string path, long offset, ThrottledStream stream)
 {
     lock (m_Lock)
     {
         m_Files.TryGetValue(path, out var f);
         if (f != null)
         {
             f.Offset = offset;
             if (f.Stream == null)
             {
                 f.Stream = stream;
             }
         }
     }
 }
Exemple #20
0
        private new async Task <XDocument> RequestDataAsync(string url)
        {
            var request = (HttpWebRequest)WebRequest.Create(url);

            request.Method          = "GET";
            request.ProtocolVersion = HttpVersion.Version11;
            request.UserAgent       =
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
            request.AllowAutoRedirect      = true;
            request.KeepAlive              = true;
            request.Pipelined              = true;
            request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
            request.ReadWriteTimeout       = shellService.Settings.TimeOut * 1000;
            request.Timeout = -1;
            ServicePointManager.DefaultConnectionLimit = 400;
            if (!string.IsNullOrEmpty(shellService.Settings.ProxyHost))
            {
                request.Proxy = new WebProxy(shellService.Settings.ProxyHost, int.Parse(shellService.Settings.ProxyPort));
            }
            else
            {
                request.Proxy = null;
            }

            var bandwidth = 2000000;

            if (shellService.Settings.LimitScanBandwidth)
            {
                bandwidth = shellService.Settings.Bandwidth;
            }

            using (var response = await request.GetResponseAsync() as HttpWebResponse)
            {
                using (
                    var stream = new ThrottledStream(response.GetResponseStream(),
                                                     (bandwidth / shellService.Settings.ParallelImages) * 1024))
                {
                    using (var buffer = new BufferedStream(stream))
                    {
                        using (var reader = new StreamReader(buffer))
                        {
                            return(XDocument.Load(reader));
                        }
                    }
                }
            }
        }
        public void TestStreamWrite()
        {
            var size           = 1024;
            var bytesPerSecond = 256; // 32 B/s
            var randomBytes    = DummyData.GenerateRandomBytes(size);

            using Stream tar = new ThrottledStream(new MemoryStream(), bytesPerSecond);
            tar.Seek(0, SeekOrigin.Begin);
            var start = Environment.TickCount64;

            tar.Write(randomBytes, 0, randomBytes.Length);

            var elapsed      = Environment.TickCount64 - start;
            var expectedTime = (size / bytesPerSecond) * 1000;

            Assert.IsTrue(elapsed >= expectedTime);
        }
Exemple #22
0
        /// <summary>
        /// Скачать файл с сервера
        /// </summary>
        /// <param name="remoteFileUri">Исходный путь и имя файла на сервере.</param>
        /// <param name="localFilePath">Путь и имя файла для загрузки в локальной файловой системе.</param>
        public async Task <bool> DownloadFileAsync(Uri remoteFileUri, string localFilePath)
        {
            bool result = false;

            Progress_Clear();
            Show_Message($"Скачивание файла {localFilePath}");

            try
            {
                using (WebDavSession session = new WebDavSession(Server, httpClientHandler, version))
                {
                    Progress <WebDavProgress> progress = new Progress <WebDavProgress>();
                    progress.ProgressChanged += Progress_ProgressChanged;

                    Stream stream = File.Create(localFilePath);
                    if (Settings.SpeedLimiter.IsLimitDownload)
                    {
                        stream = new ThrottledStream(stream, Settings.SpeedLimiter.DownloadSpeedLimit * 1024);
                    }
                    Set_Stream(stream);

                    result = await session.DownloadFileWithProgressAsync(remoteFileUri, stream, progress, CancellationToken.None);

                    if (result)
                    {
                        LogController.Info(logger, $"Файл успешно скачен: {remoteFileUri}");
                        Progress_Complete();
                    }
                    else
                    {
                        LogController.Warn(logger, $"Файл не скачен: {remoteFileUri}");
                    }

                    Close_Stream();
                    //stream.Dispose();
                }
            }
            catch (Exception e)
            {
                LogController.Error(logger, e, $"Ошибка скачивания файла: {remoteFileUri}");
            }

            Show_Message();

            return(result);
        }
Exemple #23
0
        private void TestStreamIntegrity(int streamSize, long maximumBytesPerSecond)
        {
            // arrange
            byte[] data       = DummyData.GenerateOrderedBytes(streamSize);
            byte[] copiedData = new byte[streamSize];
            using Stream stream = new ThrottledStream(new MemoryStream(), maximumBytesPerSecond);

            // act
            stream.Write(data, 0, data.Length);
            stream.Seek(0, SeekOrigin.Begin);
            stream.Read(copiedData, 0, copiedData.Length);

            // assert
            Assert.AreEqual(streamSize, data.Length);
            Assert.AreEqual(streamSize, copiedData.Length);
            Assert.IsTrue(data.SequenceEqual(copiedData));
        }
        public void TestStreamRead()
        {
            using (Stream src = new ThrottledStream(new MemoryStream(DummyData.GenerateRandomBytes(1024)), 256))
            {
                src.Seek(0, SeekOrigin.Begin);
                byte[] buf   = new byte[256];
                int    read  = 1;
                int    start = Environment.TickCount;

                while (read > 0)
                {
                    read = src.Read(buf, 0, buf.Length);
                }

                int elapsed = Environment.TickCount - start;
                Assert.GreaterOrEqual(elapsed, 4000);
            }
        }
Exemple #25
0
        public HttpResponseMessage GetFile(DtoClientFileRequest fileRequest)
        {
            var guid          = ConfigurationManager.AppSettings["ComServerUniqueId"];
            var thisComServer = new ServiceClientComServer().GetServerByGuid(guid);

            if (thisComServer == null)
            {
                Logger.Error($"Com Server With Guid {guid} Not Found");
                return(new HttpResponseMessage(HttpStatusCode.NotFound));
            }
            var maxBitRate = thisComServer.EmMaxBps;
            var basePath   = thisComServer.LocalStoragePath;

            var fullPath = Path.Combine(basePath, "software_uploads", fileRequest.ModuleGuid,
                                        fileRequest.FileName);

            if (File.Exists(fullPath))
            {
                HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
                try
                {
                    var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read);
                    if (maxBitRate == 0)
                    {
                        result.Content = new StreamContent(stream);
                    }
                    else
                    {
                        Stream throttledStream = new ThrottledStream(stream, maxBitRate);
                        result.Content = new StreamContent(throttledStream);
                    }
                    result.Content.Headers.ContentType                 = new MediaTypeHeaderValue("application/octet-stream");
                    result.Content.Headers.ContentDisposition          = new ContentDispositionHeaderValue("attachment");
                    result.Content.Headers.ContentDisposition.FileName = fileRequest.FileName;
                    return(result);
                }
                catch (Exception ex)
                {
                    Logger.Error(ex.Message);
                }
            }
            return(new HttpResponseMessage(HttpStatusCode.NotFound));
        }
Exemple #26
0
 protected virtual async Task <bool> DownloadBinaryFile(string fileLocation, string url, CancellationToken ct)
 {
     try
     {
         return(await ThrottledStream.DownloadFileWithResumeAsync(url, fileLocation,
                                                                  shellService.Settings, ct));
     }
     catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70)
     {
         Logger.Error("ManagerController:Download: {0}", ex);
         shellService.ShowError(ex, Resources.DiskFull);
         crawlerService.StopCommand.Execute(null);
         return(false);
     }
     catch
     {
         return(false);
     }
 }
Exemple #27
0
        private void TestStreamWriteSpeed(ThrottledStreamWrite writeMethod)
        {
            // arrange
            var size           = 1024;
            var bytesPerSecond = 256;                            // 256 B/s
            var tolerance      = 50;                             // 50 ms
            var expectedTime   = (size / bytesPerSecond) * 1000; // 4000 Milliseconds
            var randomBytes    = DummyData.GenerateRandomBytes(size);

            using Stream stream = new ThrottledStream(new MemoryStream(), bytesPerSecond);
            var stopWatcher = Stopwatch.StartNew();

            // act
            writeMethod(stream, randomBytes, 0, randomBytes.Length);
            stopWatcher.Stop();

            // assert
            Assert.IsTrue(stopWatcher.ElapsedMilliseconds + tolerance >= expectedTime,
                          $"actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
        }
Exemple #28
0
        public HttpResponseMessage GetFile(DtoClientFileRequest fileRequest)
        {
            var storageType = ServiceSetting.GetSettingValue(SettingStrings.StorageType);
            var basePath    = ServiceSetting.GetSettingValue(SettingStrings.StoragePath);
            var maxBitRate  = Convert.ToInt32(ConfigurationManager.AppSettings["MaximumDownloadBps"]);

            if (storageType != "Local")
            {
                basePath = ConfigurationManager.AppSettings["LocalStoragePath"];
            }

            var fullPath = Path.Combine(basePath, "software_uploads", fileRequest.ModuleGuid,
                                        fileRequest.FileName);

            if (File.Exists(fullPath))
            {
                HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
                try
                {
                    var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read);
                    if (maxBitRate == 0)
                    {
                        result.Content = new StreamContent(stream);
                    }
                    else
                    {
                        Stream throttledStream = new ThrottledStream(stream, maxBitRate);
                        result.Content = new StreamContent(throttledStream);
                    }
                    result.Content.Headers.ContentType                 = new MediaTypeHeaderValue("application/octet-stream");
                    result.Content.Headers.ContentDisposition          = new ContentDispositionHeaderValue("attachment");
                    result.Content.Headers.ContentDisposition.FileName = fileRequest.FileName;
                    return(result);
                }
                catch (Exception ex)
                {
                    Logger.Error(ex.Message);
                }
            }
            return(new HttpResponseMessage(HttpStatusCode.NotFound));
        }
        public void TestStreamRead()
        {
            var size           = 1024;
            var bytesPerSecond = 256; // 256 B/s
            var randomBytes    = DummyData.GenerateRandomBytes(size);

            using Stream src = new ThrottledStream(new MemoryStream(randomBytes), bytesPerSecond);
            src.Seek(0, SeekOrigin.Begin);
            byte[] buf   = new byte[bytesPerSecond];
            int    read  = 1;
            long   start = Environment.TickCount64;

            while (read > 0)
            {
                read = src.Read(buf, 0, buf.Length);
            }

            long elapsed      = Environment.TickCount64 - start;
            var  expectedTime = (size / bytesPerSecond) * 1000;

            Assert.IsTrue(elapsed >= expectedTime);
        }
 private bool Download(TumblrBlog blog, string fileLocation, string url)
 {
     Monitor.Enter(blog);
     if (blog.Links.Contains(url))
     {
         Monitor.Exit(blog);
         return(false);
     }
     else
     {
         Monitor.Exit(blog);
         try
         {
             using (var stream = ThrottledStream.ReadFromURLIntoStream(url, (shellService.Settings.Bandwidth / shellService.Settings.ParallelImages), shellService.Settings.TimeOut))
                 ThrottledStream.SaveStreamToDisk(stream, fileLocation);
             return(true);
         }
         catch (Exception)
         {
             return(false);
         }
     }
 }
Exemple #31
0
        public void TestFlushToCorrectOrder() {
            AutoResetEvent reset = new AutoResetEvent(false);
            List<int> flushedItems = new List<int>();

            ThrottledStream<int> stream = new ThrottledStream<int>() {
                Interval = new TimeSpan(0, 0, 0, 0, 100),
                FlushTo = items => {
                    flushedItems = items;
                    reset.Set();
                }
            };

            stream.Start();

            stream.Call(1);
            stream.Call(2);
            stream.Call(3);

            Assert.IsTrue(reset.WaitOne(500));

            Assert.AreEqual(new List<int>() { 1, 2, 3 }, flushedItems);
        }
Exemple #32
0
        private Response GetObject(string bucket, string key)
        {
            var s3Object = storage.GetObject(bucket, key);
              if (s3Object == null)
              {
            return new Response { StatusCode = HttpStatusCode.NotFound };
              }

              var stream = s3Object.Content();

              var response = new Response { StatusCode = HttpStatusCode.OK, ContentType = s3Object.ContentType };
              response.WithHeader("ETag", string.Format("\"{0}\"", s3Object.ContentMD5));
              response.WithHeader("Accept-Ranges", "bytes");
              response.Contents = x =>
              {
            var throttledStream = new ThrottledStream(stream, configuration.MaxBytesPerSecond);
            throttledStream.CopyTo(x);
              };

              return response;
        }
        public async Task<IHttpActionResult> DownloadCourseModuleClip(string clipname, ClipToSave clipToSave)
        {
            string clipUrl = string.Empty;
            // 1- get the video clip url to download.
            try
            {
                clipUrl = GetClipUrl(clipToSave);

                // 2- make sure the folders structure exist.
                var videoSaveDirectory = SetUpVideoFolderStructure(clipToSave.CourseTitle,
                    (clipToSave.ModuleIndex + 1).ToString("D2") + " - " + clipToSave.ModuleTitle,
                    (clipToSave.ClipIndex + 1).ToString("D2") + " - " + clipToSave.Title);

                // 3- download the video and report progress back.
                int receivedBytes = 0;
                long totalBytes = 0;
                var videoFileName = ((clipToSave.ClipIndex + 1).ToString("D2") + " - " + clipToSave.Title + ".mp4").ToValidFileName();
                var videoSaveLocation = videoSaveDirectory.FullName + "\\raw-" + videoFileName;

                using (var client = new WebClient())
                using (var regStream = await client.OpenReadTaskAsync(clipUrl))
                using (var stream = new ThrottledStream(regStream, 115200))
                {
                    byte[] buffer = new byte[1024];
                    totalBytes = Int32.Parse(client.ResponseHeaders[HttpResponseHeader.ContentLength]);
                    stream.MaximumBytesPerSecond = GetClipMaxDownloadSpeed(clipToSave.DurationSeconds, totalBytes);

                    using (var fileStream = File.OpenWrite(videoSaveLocation))
                    {
                        for (;;)
                        {
                            int bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length);
                            if (bytesRead == 0)
                            {
                                await Task.Yield();
                                break;
                            }

                            receivedBytes += bytesRead;
                            var progress = new ProgressArgs()
                            {
                                Id = clipToSave.Name,
                                BytesReceived = receivedBytes,
                                FileName = videoFileName,
                                TotalBytes = totalBytes,
                                IsDownloading = true,
                                Extra = new { clipToSave.ModuleIndex, clipToSave.ClipIndex }
                            };
                            fileStream.Write(buffer, 0, bytesRead);
                            var hubContext = GlobalHost.ConnectionManager.GetHubContext<ProgressHub>();
                            hubContext.Clients.All.updateProgress(progress);
                        }
                    }
                }

                // 4- save the video file.
                var inputFile = new MediaFile { Filename = videoSaveDirectory.FullName + "\\raw-" + videoFileName };
                var outputFile = new MediaFile { Filename = videoSaveDirectory.FullName + "\\" + videoFileName };

                File.Move(inputFile.Filename, outputFile.Filename);

                // 5- Create srt files
                if (Constants.SUBTITLES)
                {
                    var srtFilename = outputFile.Filename.Substring(0, outputFile.Filename.Length - 4) + ".srt";
                    var srtString = clipToSave.TranscriptClip.GetSrtString(clipToSave.DurationSeconds);
                    File.WriteAllText(srtFilename, srtString);
                }

                return Ok(new ProgressArgs()
                {
                    Id = clipToSave.Name,
                    BytesReceived = receivedBytes,
                    FileName = videoFileName,
                    TotalBytes = totalBytes,
                    IsDownloading = false,
                    Extra = new { clipToSave.ModuleIndex, clipToSave.ClipIndex }
                });
            }
            catch (Exception exception)
            {
                return HandleException(exception);
            }
        }
Exemple #34
0
        public void TestIgnoredWhenNotRunning() {
            ThrottledStream<int> stream = new ThrottledStream<int>();

            stream.Call(1);

            Assert.IsEmpty(stream.Items);
        }
Exemple #35
0
        public void TestPushedWhenRunning() {
            ThrottledStream<int> stream = new ThrottledStream<int>() {
                Running = true
            };

            stream.Call(1);

            Assert.IsNotEmpty(stream.Items);
        }
        private ServerConnectorResponse postFileStream(Uri address, FileStream file_stream)
        {
            byte[] buffer = new byte[512];
            byte[] header_bytes;
            Stream write_stream;
            int read_length = 0;
            ThrottledStream throttled_steam = null;
            var upload_args = new ServerConnectorUploadProgressChangedEventArgs();

            if(!file_stream.CanRead) // If we can not read it, what CAN we do?
                throw new ServerConnectorUploadLockedFile();

            HttpWebRequest request = prepareRequest(address);
            createPostBoundry(request);

            string content_split = "Content-Disposition: form-data; name=\"file\"; filename=\"{0}\"\r\nContent-Type: application/octet-stream\r\n\r\n";
            content_split = string.Format(content_split, Path.GetFileName(file_stream.Name));
            header_bytes = Encoding.UTF8.GetBytes(content_split);

            upload_args.total_bytes_to_send = file_stream.Length;
            request.ContentLength = boundry.boundry_start_bytes.Length + boundry.boundry_end_bytes.Length + file_stream.Length + header_bytes.Length;
            request.AllowWriteStreamBuffering = false;

            write_stream = request.GetRequestStream();

            // If the user wants to throttle, throttle the upload stream.
            if(_throttle_bytes_up != ThrottledStream.Infinite) {
                throttled_steam = new ThrottledStream(write_stream, _throttle_bytes_up * 1024);
                write_stream = throttled_steam;
            }

            write_stream.Write(boundry.boundry_start_bytes, 0, boundry.boundry_start_bytes.Length);
            write_stream.Write(header_bytes, 0, header_bytes.Length);

            while((read_length = file_stream.Read(buffer, 0, buffer.Length)) > 0) {
                write_stream.Write(buffer, 0, read_length);
                write_stream.Flush();

                if(throttle_changed && throttled_steam != null) { // If the upload throttle has changed, update the bandwidth.
                    throttled_steam.MaximumBytesPerSecond = _throttle_bytes_up * 1024;
                    throttle_changed = false;
                }

                if(cancel_request) {
                    upload_canceled.Invoke();
                    break;
                }

                // Update any events with this information
                upload_args.bytes_sent += read_length;
                upload_progress.Invoke(upload_args);
            }

            if(cancel_request == false) {
                write_stream.Write(boundry.boundry_end_bytes, 0, boundry.boundry_end_bytes.Length);
                write_stream.Close();
            }

            file_stream.Close();

            if(cancel_request) {
                cancel_request = false;
                request.Abort();

                _is_active = false;

                // Send out the queued query if any.
                requestQueueAdvance();
                return new ServerConnectorResponse();

            } else {
                return readServerResponse(request);
            }
        }
        /// <summary>
        /// Method to read the contents of the server response and return the response headers and body text.
        /// </summary>
        /// <param name="request">The request that is already in-progress.</param>
        /// <returns>First object is WebHeadersCollection containing the response headers, the second is the body text.</returns>
        private ServerConnectorResponse readServerResponse(HttpWebRequest request)
        {
            int length = 0;
            byte[] buffer = new byte[512];
            Stream response_stream;
            StringBuilder response_string = new StringBuilder();
            bool is_canceled = false;

            try {
                if(cancel_request) {
                    cancel_request = false;
                    return new ServerConnectorResponse() { canceled = true };
                }
                ThrottledStream throttled_steam = null;
                HttpWebResponse response = request.GetResponse() as HttpWebResponse;

                if(response.StatusCode != HttpStatusCode.OK) {
                    Debug.WriteLine("Query[" + request.Address.Query + "] Returned Error: " + response.StatusDescription);
                    return null;
                }

                response_stream = response.GetResponseStream();

                if(_throttle_bytes_down != ThrottledStream.Infinite){
                    throttled_steam = new ThrottledStream(response_stream, _throttle_bytes_down * 1024);
                    response_stream = throttled_steam;
                }

                while((length = response_stream.Read(buffer, 0, buffer.Length)) > 0) {
                    if(throttle_changed && throttled_steam != null) { // Check to see if the throttle has changed.  If so, change the speed.
                        throttled_steam.MaximumBytesPerSecond = _throttle_bytes_down * 1024;
                        throttle_changed = false;
                    }

                    if(cancel_request)
                        break;
                    response_string.Append(Encoding.UTF8.GetString(buffer, 0, length));
                }

                response_stream.Close();
                response.Close();
                _is_active = false;

                // Send out the queued query if any.
                requestQueueAdvance();

                if(cancel_request == true) {
                    cancel_request = false;
                    is_canceled = true;
                }

                return new ServerConnectorResponse() {
                    canceled = is_canceled,
                    headers = response.Headers,
                    body = response_string.ToString(),
                    error = false
                };
            } catch(WebException e) {
                if(cancel_request == true) {
                    cancel_request = false;
                    is_canceled = true;
                }

                return new ServerConnectorResponse() {
                    headers = (e.Response != null) ? e.Response.Headers : null,
                    canceled = is_canceled,
                    body = null,
                    error = true,
                    error_info = e.Message,
                    error_status = e.Status
                };
            }
        }
Exemple #38
-1
        public void TestStart() {
            ThrottledStream<int> stream = new ThrottledStream<int>();

            stream.Start();

            Assert.IsTrue(stream.Running);
            Assert.IsNotNull(stream.IntervalTick);
        }
Exemple #39
-1
        public void TestStop() {
            ThrottledStream<int> stream = new ThrottledStream<int>();

            stream.Stop();

            Assert.IsFalse(stream.Running);
            Assert.IsNull(stream.IntervalTick);
            Assert.IsNull(stream.FlushTo);
        }
Exemple #40
-1
        public void TestStopWhileRunning() {
            ThrottledStream<int> stream = new ThrottledStream<int>() {
                FlushTo = items => {
                    
                }
            };

            stream.Start();

            stream.Stop();

            Assert.IsFalse(stream.Running);
            Assert.IsNull(stream.IntervalTick);
            Assert.IsNull(stream.FlushTo);
        }
Exemple #41
-1
        public void TestFlushToIsCalled() {
            AutoResetEvent reset = new AutoResetEvent(false);
            bool isFlushToCalled = false;

            ThrottledStream<int> stream = new ThrottledStream<int>() {
                Interval = new TimeSpan(0, 0, 0, 0, 10),
                FlushTo = items => {
                    isFlushToCalled = true;
                    reset.Set();
                }
            };

            stream.Start();

            stream.Call(1);

            Assert.IsTrue(reset.WaitOne(500));

            Assert.IsTrue(isFlushToCalled);
        }