private void TestStreamReadSpeed(ThrottledStreamRead readMethod)
{
// arrange
var size = 1024;
var maxBytesPerSecond = 256; // 256 Byte/s
var slowExpectedTime = (size / maxBytesPerSecond) * 1000; // 4000 Milliseconds
var fastExpectedTime = slowExpectedTime * 0.75; // 3000 Milliseconds
var randomBytes = DummyData.GenerateRandomBytes(size);
var buffer = new byte[maxBytesPerSecond / 8];
var readSize = 1;
using Stream stream = new ThrottledStream(new MemoryStream(randomBytes), maxBytesPerSecond);
var stopWatcher = Stopwatch.StartNew();
// act
stream.Seek(0, SeekOrigin.Begin);
while (readSize > 0)
{
readSize = readMethod(stream, buffer, 0, buffer.Length);
}
stopWatcher.Stop();
// assert
Assert.IsTrue(stopWatcher.ElapsedMilliseconds >= fastExpectedTime, $"actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
Assert.IsTrue(stopWatcher.ElapsedMilliseconds <= slowExpectedTime, $"actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
}
private void TestStreamReadByDynamicSpeed(ThrottledStreamRead readMethod)
{
// arrange
var limitationCoefficient = 0.9; // 90%
var size = 10240; // 10KB
var maxBytesPerSecond = 1024; // 1 KByte/s
var expectedTime = (size / 2 / maxBytesPerSecond + size / 2 / (maxBytesPerSecond * 2)) * 1000 * limitationCoefficient; // 90% of 10000 Milliseconds
var randomBytes = DummyData.GenerateRandomBytes(size);
var buffer = new byte[maxBytesPerSecond / 8];
var readSize = 1;
var totalReadSize = 0L;
using ThrottledStream stream = new ThrottledStream(new MemoryStream(randomBytes), maxBytesPerSecond);
var stopWatcher = Stopwatch.StartNew();
// act
stream.Seek(0, SeekOrigin.Begin);
while (readSize > 0)
{
readSize = readMethod(stream, buffer, 0, buffer.Length);
totalReadSize += readSize;
// increase speed (2X) after downloading half size
if (totalReadSize > size / 2 && maxBytesPerSecond == stream.BandwidthLimit)
{
stream.BandwidthLimit *= 2;
}
}
stopWatcher.Stop();
// assert
Assert.IsTrue(stopWatcher.ElapsedMilliseconds >= expectedTime,
$"expected duration is: {expectedTime}ms , but actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
}
private void HandleProgress(ThrottledStream ts, long pg)
{
if (!m_taskreader.TransferProgressAsync.WaitForTask().Result)
{
throw new OperationCanceledException();
}
// Update the throttle speeds if they have changed
string tmp;
m_options.RawOptions.TryGetValue("throttle-upload", out tmp);
if (tmp != m_lastThrottleUploadValue)
{
ts.WriteSpeed = m_options.MaxUploadPrSecond;
m_lastThrottleUploadValue = tmp;
}
m_options.RawOptions.TryGetValue("throttle-download", out tmp);
if (tmp != m_lastThrottleDownloadValue)
{
ts.ReadSpeed = m_options.MaxDownloadPrSecond;
m_lastThrottleDownloadValue = tmp;
}
m_stats.UpdateBackendProgress(pg);
}
/// <summary>
/// Загрузить файл на сервер
/// !!! добавить прогресс !!!
/// </summary>
/// <param name="localFilePath">Локальный путь и имя файла для загрузки.</param>
/// <param name="remoteFileUri">Путь к файлу и имя файла на сервере.</param>
public async Task <bool> UploadFileAsync(string localFilePath, Uri remoteFileUri)
{
bool result = false;
Progress_Clear();
Show_Message($"Загрузка файла {localFilePath}");
if (quota.IsLimited)
{
FileInfo fileInfo = new FileInfo(localFilePath);
if (quota.QuotaAvailableBytes < fileInfo.Length)
{
LogController.Warn(logger, $"Недостаточно места на сервере");
Progress_Complete();
Show_Message();
return(false);
}
}
try
{
using (WebDavSession session = new WebDavSession(Server, httpClientHandler, version))
{
Progress <WebDavProgress> progress = new Progress <WebDavProgress>();
progress.ProgressChanged += Progress_ProgressChanged;
session.Timeout = TimeSpan.FromDays(1);
Stream stream = File.OpenRead(localFilePath);
if (Settings.SpeedLimiter.IsLimitUpload)
{
stream = new ThrottledStream(stream, Settings.SpeedLimiter.UploadSpeedLimit * 1024);
}
Set_Stream(stream);
string contentType = MimeMapping.GetMimeMapping(localFilePath);
result = await session.UploadFileWithProgressAsync(remoteFileUri, stream, contentType, progress, CancellationToken.None);
if (result)
{
LogController.Info(logger, $"Файл успешно загружен: {localFilePath}");
Progress_Complete();
}
else
{
LogController.Warn(logger, $"Файл не загружен: {localFilePath}");
}
Close_Stream();
//stream.Dispose();
}
}
catch (Exception e)
{
LogController.Error(logger, e, $"Ошибка загрузки файла: {localFilePath}");
}
Show_Message();
return(result);
}
private void DownloadFile(FileLink fileLink, WebClient client)
{
Console.WriteLine("File {0} status - started.", fileLink.Name);
var sppedInBps = _speedInKbps * 1024;
_timer.Start();
using (var stream = client.OpenRead(fileLink.HttpAddress))
{
var throttledStream = new ThrottledStream(stream, sppedInBps);
var fileName = _outputPath + fileLink.Name;
using (var file = File.Create(fileName))
{
var buffer = new byte[BufferSize];
var readCount = throttledStream.Read(buffer, 0, BufferSize);
while (readCount > 0)
{
file.Write(buffer, 0, readCount);
readCount = throttledStream.Read(buffer, 0, BufferSize);
}
}
throttledStream.Close();
}
_timer.Stop();
Console.WriteLine("File {0} status - downloaded in {1} seconds.", fileLink.Name, _timer.ElapsedMilliseconds / 1000);
_timer.Reset();
}
public void TestStreamIntegrity()
{
using (Stream tar = new ThrottledStream(new MemoryStream(), 100))
{
byte[] buf = DummyData.GenerateOrderedBytes(500);
tar.Write(buf, 0, buf.Length);
tar.Seek(0, SeekOrigin.Begin);
byte[] buf2 = new byte[500];
tar.Read(buf2, 0, buf2.Length);
Assert.IsTrue(buf.SequenceEqual(buf2));
}
using (Stream tar = new ThrottledStream(new MemoryStream()))
{
byte[] buf = DummyData.GenerateOrderedBytes(4096);
tar.Write(buf, 0, buf.Length);
tar.Seek(0, SeekOrigin.Begin);
byte[] buf2 = new byte[4096];
tar.Read(buf2, 0, buf2.Length);
Assert.IsTrue(buf.SequenceEqual(buf2));
}
using (Stream tar = new ThrottledStream(new MemoryStream(), 77))
{
byte[] buf = DummyData.GenerateOrderedBytes(247);
tar.Write(buf, 0, buf.Length);
tar.Seek(0, SeekOrigin.Begin);
byte[] buf2 = new byte[247];
tar.Read(buf2, 0, buf2.Length);
Assert.IsTrue(buf.SequenceEqual(buf2));
}
}
public static void ThrottledStreamRead()
{
byte[] sourceBuffer = { 0x10, 0x20, 0x30, 0x40, 0x50 };
byte[] destinationBuffer = new byte[sourceBuffer.Length + 1];
const int offset = 1;
const int bytesToRead = 3;
using (MemoryStream baseStream = new MemoryStream(sourceBuffer))
{
const int readSpeed = 1;
const int writeSpeed = 1;
ThrottledStream throttledStream = new ThrottledStream(baseStream, readSpeed, writeSpeed);
int bytesRead = throttledStream.Read(destinationBuffer, offset, bytesToRead);
Assert.AreEqual(bytesToRead, bytesRead);
for (int k = 0; k < destinationBuffer.Length; k++)
{
if (offset <= k && k < offset + bytesToRead)
{
Assert.AreEqual(sourceBuffer[k - offset], destinationBuffer[k]);
}
else
{
Assert.AreEqual(default(byte), destinationBuffer[k]);
}
}
}
}
private Datamodels.TumblrJson RequestData(string url, string authHeaders)
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "GET";
request.ServicePoint.Expect100Continue = false;
request.ContentType = "x-www-from-urlencoded";
if (authHeaders != null)
{
// add OAuth header
request.Headers["Authorization"] = authHeaders;
}
var jsserializer = new JavaScriptSerializer();
using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
{
ThrottledStream stream = new ThrottledStream(response.GetResponseStream(), (shellService.Settings.Bandwidth / shellService.Settings.ParallelImages) * 1024);
StreamReader reader = new StreamReader(stream);
DataModels.TumblrJson data = jsserializer.Deserialize <DataModels.TumblrJson>(reader.ReadToEnd());
if (data.meta.status == 200)
{
return(data);
}
return(null);
}
}
catch (WebException ex)
{
return(null);
}
}
private Response GetObject(string bucket, string key)
{
var s3Object = storage.GetObject(bucket, key);
if (s3Object == null)
{
return(new Response {
StatusCode = HttpStatusCode.NotFound
});
}
var stream = s3Object.Content();
var response = new Response {
StatusCode = HttpStatusCode.OK, ContentType = s3Object.ContentType
};
response.WithHeader("ETag", string.Format("\"{0}\"", s3Object.ContentMD5));
response.WithHeader("Accept-Ranges", "bytes");
response.Contents = x =>
{
var throttledStream = new ThrottledStream(stream, configuration.MaxBytesPerSecond);
throttledStream.CopyTo(x);
};
return(response);
}
private void TestStreamReadSpeed(ThrottledStreamRead readMethod)
{
// arrange
var limitationCoefficient = 0.8; // 80%
var size = 10240; // 10KB
var maxBytesPerSecond = 1024; // 1024 Byte/s
var expectedTime = (size / maxBytesPerSecond) * 1000 * limitationCoefficient; // 80% of 10000 Milliseconds
var randomBytes = DummyData.GenerateRandomBytes(size);
var buffer = new byte[maxBytesPerSecond / 8];
var readSize = 1;
using Stream stream = new ThrottledStream(new MemoryStream(randomBytes), maxBytesPerSecond);
var stopWatcher = Stopwatch.StartNew();
// act
stream.Seek(0, SeekOrigin.Begin);
while (readSize > 0)
{
readSize = readMethod(stream, buffer, 0, buffer.Length);
}
stopWatcher.Stop();
// assert
Assert.IsTrue(stopWatcher.ElapsedMilliseconds >= expectedTime,
$"expected duration is: {expectedTime}ms , but actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
}
public static void ThrottledStreamWrite()
{
byte[] initialBuffer = { 0x10, 0x20, 0x30, 0x40, 0x50 };
byte[] source = { 0x60, 0x70, 0x80, 0x90 };
const int offset = 1;
const int bytesToWrite = 3;
using (MemoryStream baseStream = new MemoryStream(initialBuffer))
{
const int readSpeed = 1;
const int writeSpeed = 1;
ThrottledStream throttledStream = new ThrottledStream(baseStream, readSpeed, writeSpeed);
throttledStream.Write(source, offset, bytesToWrite);
byte[] result = baseStream.ToArray();
for (int k = 0; k < result.Length; k++)
{
if (k < bytesToWrite)
{
Assert.AreEqual(source[offset + k], result[k]);
}
else
{
Assert.AreEqual(initialBuffer[k], result[k]);
}
}
}
}
public void TestZeroBandwidth()
{
// arrange
int maximumBytesPerSecond = 0;
// act
using var throttledStream = new ThrottledStream(new MemoryStream(), maximumBytesPerSecond);
// assert
Assert.AreEqual(long.MaxValue, throttledStream.BandwidthLimit);
}
internal Peer(Stream stream, IConnection connection)
{
Connection = connection;
SendSpeedWatcher = new SpeedWatcher();
ReceiveSpeedWatcher = new SpeedWatcher();
SendBandwidthController = new UnlimitedBandwidthController();
ReceiveBandwidthController = new UnlimitedBandwidthController();
Statistics = new PeerStat();
Stream = new ThrottledStream(stream, ReceiveBandwidthController, SendBandwidthController);
Uri = new Uri("tcp://" + EndPoint.Address + ':' + EndPoint.Port);
}
public HttpResponseMessage GetImagingFile(DtoImageFileRequest fileRequest)
{
var guid = ConfigurationManager.AppSettings["ComServerUniqueId"];
var thisComServer = new ServiceClientComServer().GetServerByGuid(guid);
if (thisComServer == null)
{
Logger.Error($"Com Server With Guid {guid} Not Found");
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
var profile = new ServiceImageProfile().ReadProfile(fileRequest.profileId);
if (profile == null)
{
Logger.Error($"Image Profile Not Found: {fileRequest.profileId}");
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
var maxBitRate = thisComServer.ImagingMaxBps;
var basePath = thisComServer.LocalStoragePath;
var fullPath = Path.Combine(basePath, "images", profile.Image.Name, $"hd{fileRequest.hdNumber}",
fileRequest.fileName);
if (File.Exists(fullPath))
{
HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
try
{
var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read);
if (maxBitRate == 0)
{
result.Content = new StreamContent(stream);
}
else
{
Stream throttledStream = new ThrottledStream(stream, maxBitRate);
result.Content = new StreamContent(throttledStream);
}
result.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("inline");
result.Content.Headers.ContentDisposition.FileName = fileRequest.fileName;
return(result);
}
catch (Exception ex)
{
Logger.Error(ex.Message);
}
}
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
public void TestNegativeBandwidth()
{
// arrange
int maximumBytesPerSecond = -1;
// act
void CreateThrottledStream()
{
using var throttledStream = new ThrottledStream(new MemoryStream(), maximumBytesPerSecond);
}
// assert
Assert.ThrowsException <ArgumentOutOfRangeException>(CreateThrottledStream);
}
public void TestStreamWrite()
{
using (Stream tar = new ThrottledStream(new MemoryStream(), 256))
{
tar.Seek(0, SeekOrigin.Begin);
byte[] buf = DummyData.GenerateRandomBytes(1024);
int start = Environment.TickCount;
tar.Write(buf, 0, buf.Length);
int elapsed = Environment.TickCount - start;
Assert.GreaterOrEqual(elapsed, 4000);
}
}
private async Task <SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string[] pathParts, SyncTarget target, SyncOptions options, IProgress <double> progress, CancellationToken cancellationToken)
{
_logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, string.Join("/", pathParts));
using (var fileStream = _fileSystem.GetFileStream(inputPath, FileMode.Open, FileAccess.Read, FileShare.Read, true))
{
Stream stream = fileStream;
if (options.UploadSpeedLimitBytes > 0 && provider is IRemoteSyncProvider)
{
stream = new ThrottledStream(stream, options.UploadSpeedLimitBytes);
}
return(await provider.SendFile(stream, pathParts, target, progress, cancellationToken).ConfigureAwait(false));
}
}
/// <summary>
/// Throttles for the specified buffer size in bytes.
/// </summary>
/// <param name="maximumBytesPerSecond">The maximum bytes per second that can be transferred through the base stream. Specify zero to disable this feature.</param>
public void LimitTransferSpeed(long maximumBytesPerSecond)
{
if (DataStream == null)
{
throw new ArgumentNullException("DataStream");
}
if (DataStream is ThrottledStream)
{
((ThrottledStream)DataStream).MaximumBytesPerSecond = maximumBytesPerSecond;
}
else
{
DataStream = new ThrottledStream(DataStream, maximumBytesPerSecond);
}
}
public void UpdateFileProgress(string path, long offset, ThrottledStream stream)
{
lock (m_Lock)
{
m_Files.TryGetValue(path, out var f);
if (f != null)
{
f.Offset = offset;
if (f.Stream == null)
{
f.Stream = stream;
}
}
}
}
private new async Task <XDocument> RequestDataAsync(string url)
{
var request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "GET";
request.ProtocolVersion = HttpVersion.Version11;
request.UserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
request.AllowAutoRedirect = true;
request.KeepAlive = true;
request.Pipelined = true;
request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
request.ReadWriteTimeout = shellService.Settings.TimeOut * 1000;
request.Timeout = -1;
ServicePointManager.DefaultConnectionLimit = 400;
if (!string.IsNullOrEmpty(shellService.Settings.ProxyHost))
{
request.Proxy = new WebProxy(shellService.Settings.ProxyHost, int.Parse(shellService.Settings.ProxyPort));
}
else
{
request.Proxy = null;
}
var bandwidth = 2000000;
if (shellService.Settings.LimitScanBandwidth)
{
bandwidth = shellService.Settings.Bandwidth;
}
using (var response = await request.GetResponseAsync() as HttpWebResponse)
{
using (
var stream = new ThrottledStream(response.GetResponseStream(),
(bandwidth / shellService.Settings.ParallelImages) * 1024))
{
using (var buffer = new BufferedStream(stream))
{
using (var reader = new StreamReader(buffer))
{
return(XDocument.Load(reader));
}
}
}
}
}
public void TestStreamWrite()
{
var size = 1024;
var bytesPerSecond = 256; // 32 B/s
var randomBytes = DummyData.GenerateRandomBytes(size);
using Stream tar = new ThrottledStream(new MemoryStream(), bytesPerSecond);
tar.Seek(0, SeekOrigin.Begin);
var start = Environment.TickCount64;
tar.Write(randomBytes, 0, randomBytes.Length);
var elapsed = Environment.TickCount64 - start;
var expectedTime = (size / bytesPerSecond) * 1000;
Assert.IsTrue(elapsed >= expectedTime);
}
/// <summary>
/// Скачать файл с сервера
/// </summary>
/// <param name="remoteFileUri">Исходный путь и имя файла на сервере.</param>
/// <param name="localFilePath">Путь и имя файла для загрузки в локальной файловой системе.</param>
public async Task <bool> DownloadFileAsync(Uri remoteFileUri, string localFilePath)
{
bool result = false;
Progress_Clear();
Show_Message($"Скачивание файла {localFilePath}");
try
{
using (WebDavSession session = new WebDavSession(Server, httpClientHandler, version))
{
Progress <WebDavProgress> progress = new Progress <WebDavProgress>();
progress.ProgressChanged += Progress_ProgressChanged;
Stream stream = File.Create(localFilePath);
if (Settings.SpeedLimiter.IsLimitDownload)
{
stream = new ThrottledStream(stream, Settings.SpeedLimiter.DownloadSpeedLimit * 1024);
}
Set_Stream(stream);
result = await session.DownloadFileWithProgressAsync(remoteFileUri, stream, progress, CancellationToken.None);
if (result)
{
LogController.Info(logger, $"Файл успешно скачен: {remoteFileUri}");
Progress_Complete();
}
else
{
LogController.Warn(logger, $"Файл не скачен: {remoteFileUri}");
}
Close_Stream();
//stream.Dispose();
}
}
catch (Exception e)
{
LogController.Error(logger, e, $"Ошибка скачивания файла: {remoteFileUri}");
}
Show_Message();
return(result);
}
private void TestStreamIntegrity(int streamSize, long maximumBytesPerSecond)
{
// arrange
byte[] data = DummyData.GenerateOrderedBytes(streamSize);
byte[] copiedData = new byte[streamSize];
using Stream stream = new ThrottledStream(new MemoryStream(), maximumBytesPerSecond);
// act
stream.Write(data, 0, data.Length);
stream.Seek(0, SeekOrigin.Begin);
stream.Read(copiedData, 0, copiedData.Length);
// assert
Assert.AreEqual(streamSize, data.Length);
Assert.AreEqual(streamSize, copiedData.Length);
Assert.IsTrue(data.SequenceEqual(copiedData));
}
public void TestStreamRead()
{
using (Stream src = new ThrottledStream(new MemoryStream(DummyData.GenerateRandomBytes(1024)), 256))
{
src.Seek(0, SeekOrigin.Begin);
byte[] buf = new byte[256];
int read = 1;
int start = Environment.TickCount;
while (read > 0)
{
read = src.Read(buf, 0, buf.Length);
}
int elapsed = Environment.TickCount - start;
Assert.GreaterOrEqual(elapsed, 4000);
}
}
public HttpResponseMessage GetFile(DtoClientFileRequest fileRequest)
{
var guid = ConfigurationManager.AppSettings["ComServerUniqueId"];
var thisComServer = new ServiceClientComServer().GetServerByGuid(guid);
if (thisComServer == null)
{
Logger.Error($"Com Server With Guid {guid} Not Found");
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
var maxBitRate = thisComServer.EmMaxBps;
var basePath = thisComServer.LocalStoragePath;
var fullPath = Path.Combine(basePath, "software_uploads", fileRequest.ModuleGuid,
fileRequest.FileName);
if (File.Exists(fullPath))
{
HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
try
{
var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read);
if (maxBitRate == 0)
{
result.Content = new StreamContent(stream);
}
else
{
Stream throttledStream = new ThrottledStream(stream, maxBitRate);
result.Content = new StreamContent(throttledStream);
}
result.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment");
result.Content.Headers.ContentDisposition.FileName = fileRequest.FileName;
return(result);
}
catch (Exception ex)
{
Logger.Error(ex.Message);
}
}
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
protected virtual async Task <bool> DownloadBinaryFile(string fileLocation, string url, CancellationToken ct)
{
try
{
return(await ThrottledStream.DownloadFileWithResumeAsync(url, fileLocation,
shellService.Settings, ct));
}
catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70)
{
Logger.Error("ManagerController:Download: {0}", ex);
shellService.ShowError(ex, Resources.DiskFull);
crawlerService.StopCommand.Execute(null);
return(false);
}
catch
{
return(false);
}
}
private void TestStreamWriteSpeed(ThrottledStreamWrite writeMethod)
{
// arrange
var size = 1024;
var bytesPerSecond = 256; // 256 B/s
var tolerance = 50; // 50 ms
var expectedTime = (size / bytesPerSecond) * 1000; // 4000 Milliseconds
var randomBytes = DummyData.GenerateRandomBytes(size);
using Stream stream = new ThrottledStream(new MemoryStream(), bytesPerSecond);
var stopWatcher = Stopwatch.StartNew();
// act
writeMethod(stream, randomBytes, 0, randomBytes.Length);
stopWatcher.Stop();
// assert
Assert.IsTrue(stopWatcher.ElapsedMilliseconds + tolerance >= expectedTime,
$"actual duration is: {stopWatcher.ElapsedMilliseconds}ms");
}
public HttpResponseMessage GetFile(DtoClientFileRequest fileRequest)
{
var storageType = ServiceSetting.GetSettingValue(SettingStrings.StorageType);
var basePath = ServiceSetting.GetSettingValue(SettingStrings.StoragePath);
var maxBitRate = Convert.ToInt32(ConfigurationManager.AppSettings["MaximumDownloadBps"]);
if (storageType != "Local")
{
basePath = ConfigurationManager.AppSettings["LocalStoragePath"];
}
var fullPath = Path.Combine(basePath, "software_uploads", fileRequest.ModuleGuid,
fileRequest.FileName);
if (File.Exists(fullPath))
{
HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
try
{
var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read);
if (maxBitRate == 0)
{
result.Content = new StreamContent(stream);
}
else
{
Stream throttledStream = new ThrottledStream(stream, maxBitRate);
result.Content = new StreamContent(throttledStream);
}
result.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment");
result.Content.Headers.ContentDisposition.FileName = fileRequest.FileName;
return(result);
}
catch (Exception ex)
{
Logger.Error(ex.Message);
}
}
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
public void TestStreamRead()
{
var size = 1024;
var bytesPerSecond = 256; // 256 B/s
var randomBytes = DummyData.GenerateRandomBytes(size);
using Stream src = new ThrottledStream(new MemoryStream(randomBytes), bytesPerSecond);
src.Seek(0, SeekOrigin.Begin);
byte[] buf = new byte[bytesPerSecond];
int read = 1;
long start = Environment.TickCount64;
while (read > 0)
{
read = src.Read(buf, 0, buf.Length);
}
long elapsed = Environment.TickCount64 - start;
var expectedTime = (size / bytesPerSecond) * 1000;
Assert.IsTrue(elapsed >= expectedTime);
}
private bool Download(TumblrBlog blog, string fileLocation, string url)
{
Monitor.Enter(blog);
if (blog.Links.Contains(url))
{
Monitor.Exit(blog);
return(false);
}
else
{
Monitor.Exit(blog);
try
{
using (var stream = ThrottledStream.ReadFromURLIntoStream(url, (shellService.Settings.Bandwidth / shellService.Settings.ParallelImages), shellService.Settings.TimeOut))
ThrottledStream.SaveStreamToDisk(stream, fileLocation);
return(true);
}
catch (Exception)
{
return(false);
}
}
}
public void TestFlushToCorrectOrder() {
AutoResetEvent reset = new AutoResetEvent(false);
List<int> flushedItems = new List<int>();
ThrottledStream<int> stream = new ThrottledStream<int>() {
Interval = new TimeSpan(0, 0, 0, 0, 100),
FlushTo = items => {
flushedItems = items;
reset.Set();
}
};
stream.Start();
stream.Call(1);
stream.Call(2);
stream.Call(3);
Assert.IsTrue(reset.WaitOne(500));
Assert.AreEqual(new List<int>() { 1, 2, 3 }, flushedItems);
}
private Response GetObject(string bucket, string key)
{
var s3Object = storage.GetObject(bucket, key);
if (s3Object == null)
{
return new Response { StatusCode = HttpStatusCode.NotFound };
}
var stream = s3Object.Content();
var response = new Response { StatusCode = HttpStatusCode.OK, ContentType = s3Object.ContentType };
response.WithHeader("ETag", string.Format("\"{0}\"", s3Object.ContentMD5));
response.WithHeader("Accept-Ranges", "bytes");
response.Contents = x =>
{
var throttledStream = new ThrottledStream(stream, configuration.MaxBytesPerSecond);
throttledStream.CopyTo(x);
};
return response;
}
public async Task<IHttpActionResult> DownloadCourseModuleClip(string clipname, ClipToSave clipToSave)
{
string clipUrl = string.Empty;
// 1- get the video clip url to download.
try
{
clipUrl = GetClipUrl(clipToSave);
// 2- make sure the folders structure exist.
var videoSaveDirectory = SetUpVideoFolderStructure(clipToSave.CourseTitle,
(clipToSave.ModuleIndex + 1).ToString("D2") + " - " + clipToSave.ModuleTitle,
(clipToSave.ClipIndex + 1).ToString("D2") + " - " + clipToSave.Title);
// 3- download the video and report progress back.
int receivedBytes = 0;
long totalBytes = 0;
var videoFileName = ((clipToSave.ClipIndex + 1).ToString("D2") + " - " + clipToSave.Title + ".mp4").ToValidFileName();
var videoSaveLocation = videoSaveDirectory.FullName + "\\raw-" + videoFileName;
using (var client = new WebClient())
using (var regStream = await client.OpenReadTaskAsync(clipUrl))
using (var stream = new ThrottledStream(regStream, 115200))
{
byte[] buffer = new byte[1024];
totalBytes = Int32.Parse(client.ResponseHeaders[HttpResponseHeader.ContentLength]);
stream.MaximumBytesPerSecond = GetClipMaxDownloadSpeed(clipToSave.DurationSeconds, totalBytes);
using (var fileStream = File.OpenWrite(videoSaveLocation))
{
for (;;)
{
int bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length);
if (bytesRead == 0)
{
await Task.Yield();
break;
}
receivedBytes += bytesRead;
var progress = new ProgressArgs()
{
Id = clipToSave.Name,
BytesReceived = receivedBytes,
FileName = videoFileName,
TotalBytes = totalBytes,
IsDownloading = true,
Extra = new { clipToSave.ModuleIndex, clipToSave.ClipIndex }
};
fileStream.Write(buffer, 0, bytesRead);
var hubContext = GlobalHost.ConnectionManager.GetHubContext<ProgressHub>();
hubContext.Clients.All.updateProgress(progress);
}
}
}
// 4- save the video file.
var inputFile = new MediaFile { Filename = videoSaveDirectory.FullName + "\\raw-" + videoFileName };
var outputFile = new MediaFile { Filename = videoSaveDirectory.FullName + "\\" + videoFileName };
File.Move(inputFile.Filename, outputFile.Filename);
// 5- Create srt files
if (Constants.SUBTITLES)
{
var srtFilename = outputFile.Filename.Substring(0, outputFile.Filename.Length - 4) + ".srt";
var srtString = clipToSave.TranscriptClip.GetSrtString(clipToSave.DurationSeconds);
File.WriteAllText(srtFilename, srtString);
}
return Ok(new ProgressArgs()
{
Id = clipToSave.Name,
BytesReceived = receivedBytes,
FileName = videoFileName,
TotalBytes = totalBytes,
IsDownloading = false,
Extra = new { clipToSave.ModuleIndex, clipToSave.ClipIndex }
});
}
catch (Exception exception)
{
return HandleException(exception);
}
}
public void TestIgnoredWhenNotRunning() {
ThrottledStream<int> stream = new ThrottledStream<int>();
stream.Call(1);
Assert.IsEmpty(stream.Items);
}
public void TestPushedWhenRunning() {
ThrottledStream<int> stream = new ThrottledStream<int>() {
Running = true
};
stream.Call(1);
Assert.IsNotEmpty(stream.Items);
}
private ServerConnectorResponse postFileStream(Uri address, FileStream file_stream)
{
byte[] buffer = new byte[512];
byte[] header_bytes;
Stream write_stream;
int read_length = 0;
ThrottledStream throttled_steam = null;
var upload_args = new ServerConnectorUploadProgressChangedEventArgs();
if(!file_stream.CanRead) // If we can not read it, what CAN we do?
throw new ServerConnectorUploadLockedFile();
HttpWebRequest request = prepareRequest(address);
createPostBoundry(request);
string content_split = "Content-Disposition: form-data; name=\"file\"; filename=\"{0}\"\r\nContent-Type: application/octet-stream\r\n\r\n";
content_split = string.Format(content_split, Path.GetFileName(file_stream.Name));
header_bytes = Encoding.UTF8.GetBytes(content_split);
upload_args.total_bytes_to_send = file_stream.Length;
request.ContentLength = boundry.boundry_start_bytes.Length + boundry.boundry_end_bytes.Length + file_stream.Length + header_bytes.Length;
request.AllowWriteStreamBuffering = false;
write_stream = request.GetRequestStream();
// If the user wants to throttle, throttle the upload stream.
if(_throttle_bytes_up != ThrottledStream.Infinite) {
throttled_steam = new ThrottledStream(write_stream, _throttle_bytes_up * 1024);
write_stream = throttled_steam;
}
write_stream.Write(boundry.boundry_start_bytes, 0, boundry.boundry_start_bytes.Length);
write_stream.Write(header_bytes, 0, header_bytes.Length);
while((read_length = file_stream.Read(buffer, 0, buffer.Length)) > 0) {
write_stream.Write(buffer, 0, read_length);
write_stream.Flush();
if(throttle_changed && throttled_steam != null) { // If the upload throttle has changed, update the bandwidth.
throttled_steam.MaximumBytesPerSecond = _throttle_bytes_up * 1024;
throttle_changed = false;
}
if(cancel_request) {
upload_canceled.Invoke();
break;
}
// Update any events with this information
upload_args.bytes_sent += read_length;
upload_progress.Invoke(upload_args);
}
if(cancel_request == false) {
write_stream.Write(boundry.boundry_end_bytes, 0, boundry.boundry_end_bytes.Length);
write_stream.Close();
}
file_stream.Close();
if(cancel_request) {
cancel_request = false;
request.Abort();
_is_active = false;
// Send out the queued query if any.
requestQueueAdvance();
return new ServerConnectorResponse();
} else {
return readServerResponse(request);
}
}
/// <summary>
/// Method to read the contents of the server response and return the response headers and body text.
/// </summary>
/// <param name="request">The request that is already in-progress.</param>
/// <returns>First object is WebHeadersCollection containing the response headers, the second is the body text.</returns>
private ServerConnectorResponse readServerResponse(HttpWebRequest request)
{
int length = 0;
byte[] buffer = new byte[512];
Stream response_stream;
StringBuilder response_string = new StringBuilder();
bool is_canceled = false;
try {
if(cancel_request) {
cancel_request = false;
return new ServerConnectorResponse() { canceled = true };
}
ThrottledStream throttled_steam = null;
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
if(response.StatusCode != HttpStatusCode.OK) {
Debug.WriteLine("Query[" + request.Address.Query + "] Returned Error: " + response.StatusDescription);
return null;
}
response_stream = response.GetResponseStream();
if(_throttle_bytes_down != ThrottledStream.Infinite){
throttled_steam = new ThrottledStream(response_stream, _throttle_bytes_down * 1024);
response_stream = throttled_steam;
}
while((length = response_stream.Read(buffer, 0, buffer.Length)) > 0) {
if(throttle_changed && throttled_steam != null) { // Check to see if the throttle has changed. If so, change the speed.
throttled_steam.MaximumBytesPerSecond = _throttle_bytes_down * 1024;
throttle_changed = false;
}
if(cancel_request)
break;
response_string.Append(Encoding.UTF8.GetString(buffer, 0, length));
}
response_stream.Close();
response.Close();
_is_active = false;
// Send out the queued query if any.
requestQueueAdvance();
if(cancel_request == true) {
cancel_request = false;
is_canceled = true;
}
return new ServerConnectorResponse() {
canceled = is_canceled,
headers = response.Headers,
body = response_string.ToString(),
error = false
};
} catch(WebException e) {
if(cancel_request == true) {
cancel_request = false;
is_canceled = true;
}
return new ServerConnectorResponse() {
headers = (e.Response != null) ? e.Response.Headers : null,
canceled = is_canceled,
body = null,
error = true,
error_info = e.Message,
error_status = e.Status
};
}
}
public void TestStart() {
ThrottledStream<int> stream = new ThrottledStream<int>();
stream.Start();
Assert.IsTrue(stream.Running);
Assert.IsNotNull(stream.IntervalTick);
}
public void TestStop() {
ThrottledStream<int> stream = new ThrottledStream<int>();
stream.Stop();
Assert.IsFalse(stream.Running);
Assert.IsNull(stream.IntervalTick);
Assert.IsNull(stream.FlushTo);
}
public void TestStopWhileRunning() {
ThrottledStream<int> stream = new ThrottledStream<int>() {
FlushTo = items => {
}
};
stream.Start();
stream.Stop();
Assert.IsFalse(stream.Running);
Assert.IsNull(stream.IntervalTick);
Assert.IsNull(stream.FlushTo);
}
public void TestFlushToIsCalled() {
AutoResetEvent reset = new AutoResetEvent(false);
bool isFlushToCalled = false;
ThrottledStream<int> stream = new ThrottledStream<int>() {
Interval = new TimeSpan(0, 0, 0, 0, 10),
FlushTo = items => {
isFlushToCalled = true;
reset.Set();
}
};
stream.Start();
stream.Call(1);
Assert.IsTrue(reset.WaitOne(500));
Assert.IsTrue(isFlushToCalled);
}
