protected TestUnitResult Upload(ITransferRequest transferRequest)
{
if (transferRequest == null)
{
return(null);
}
List <IMetric> metrics = new List <IMetric>();
metrics.Add(new StringMetric(MetricName.url, transferRequest.RequestUri.ToString(), ""));
int taskId = Task.CurrentId.Value;
DateTimeOffset timeStart = DateTimeOffset.UtcNow;
metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime"));
List <ItemAccessMethod> methods = new List <ItemAccessMethod>()
{
transferRequest.Method
};
if (methods.Count() == 0)
{
log.WarnFormat("No upload methods found. Skipping upload.");
var tcr1 = new TestUnitResult(metrics, TestUnitResultStatus.NotStarted);
tcr1.State = transferRequest;
metrics.Add(new DateTimeMetric(MetricName.endTime, DateTimeOffset.UtcNow, "dateTime"));
return(tcr1);
}
var method = methods.First();
long respTime = 0;
Random rng = new Random();
long totalSize = Convert.ToInt64(transferRequest.ContentLength);
log.DebugFormat("Upload {1} to {0} ({2}) ...", transferRequest.RequestUri, BytesToString(Convert.ToInt64(totalSize)), transferRequest.Method);
Stopwatch stopWatchUploadElaspedTime = new Stopwatch();
metrics.Add(new LongMetric(MetricName.beginGetResponseTime, DateTimeOffset.UtcNow.Ticks, "ticks"));
Stopwatch sw = new Stopwatch();
sw.Start();
var uploadStreamTask = transferRequest.GetRequestStreamAsync();
uploadStreamTask.ContinueWith(streamTask =>
{
respTime = sw.ElapsedMilliseconds;
Stream stream = streamTask.Result;
byte[] buf = new byte[4096];
double progessPct = 0;
long totalByteCounter = 0;
long intermediateCounter = 0;
stopWatchUploadElaspedTime.Start();
while (totalByteCounter < totalSize)
{
rng.NextBytes(buf);
int length = buf.Length;
if (totalSize < totalByteCounter + length)
{
length = Convert.ToInt32(totalSize - totalByteCounter);
}
stream.Write(buf, 0, buf.Length);
totalByteCounter += buf.Length;
intermediateCounter += buf.Length;
double totalPct = ((double)totalByteCounter / totalSize) * 100;
if ((Math.Abs(progessPct - totalPct) >= 1) || intermediateCounter >= 10485760)
{
progessPct = totalPct;
double bytespersec = (double)totalByteCounter / ((double)stopWatchUploadElaspedTime.ElapsedMilliseconds / 1000);
log.DebugFormat("[{2}] Uploaded {0}% [{1}/s]", totalPct.ToString("F1"), BytesToString((long)bytespersec), taskId);
intermediateCounter = 0;
}
if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2)
{
// Test Breaker at 20MB )
log.Warn("!!!!!TEST DOWNLOAD BREAKER!!!!");
break;
}
}
stream.Close();
stopWatchUploadElaspedTime.Stop();
});
try
{
uploadStreamTask.Wait();
}
catch (Exception e)
{
Exception ie = e;
while (ie.InnerException != null)
{
ie = ie.InnerException;
}
if (ie is WebException)
{
WebException we = ie as WebException;
log.DebugFormat("[{0}] Error uploading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, transferRequest.RequestUri, we.Status.ToString(), method);
if (we.Response is HttpWebResponse)
{
metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), ""));
}
metrics.Add(new ExceptionMetric(we));
}
else
{
log.DebugFormat("[{0}] Error during upload {2}. Exception: {1}", Task.CurrentId, ie.Message, transferRequest.RequestUri);
log.Debug(ie.StackTrace);
metrics.Add(new ExceptionMetric(ie));
}
}
var uploadTask = transferRequest.GetResponseAsync().ContinueWith(resp =>
{
sw.Stop();
using (ITransferResponse response = resp.Result)
{
log.InfoFormat("[{1}] > {3} Status Code {0} ({2}ms)", response.StatusCode, taskId, respTime, method);
metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)response.StatusCode, response.StatusDescription), ""));
if (!(response.StatusCode == TransferStatusCode.OK || response.StatusCode == TransferStatusCode.Accepted || response.StatusCode == TransferStatusCode.Created))
{
log.DebugFormat("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription);
metrics.Add(new ExceptionMetric(new Exception(
string.Format("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription))));
}
else
{
log.InfoFormat("[{0}] < OK. Content Type {1}", taskId, response.ContentType);
}
}
});
try
{
uploadTask.Wait();
}
catch (Exception e)
{
Exception ie = e;
while (ie.InnerException != null)
{
ie = ie.InnerException;
}
if (ie is WebException)
{
WebException we = ie as WebException;
log.DebugFormat("[{0}] Error uploading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, transferRequest.RequestUri, we.Status.ToString(), method);
if (we.Response is HttpWebResponse)
{
metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), ""));
}
metrics.Add(new ExceptionMetric(we));
}
else
{
log.DebugFormat("[{0}] Error during upload {2}. Exception: {1}", Task.CurrentId, ie.Message, transferRequest.RequestUri);
log.Debug(ie.StackTrace);
metrics.Add(new ExceptionMetric(ie));
}
}
DateTimeOffset timeStop = DateTimeOffset.UtcNow;
metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime"));
metrics.Add(new LongMetric(MetricName.endGetResponseTime, DateTime.UtcNow.Ticks, "ticks"));
metrics.Add(new LongMetric(MetricName.responseTime, respTime, "ms"));
metrics.Add(new LongMetric(MetricName.size, totalSize, "bytes"));
metrics.Add(new LongMetric(MetricName.downloadElapsedTime, stopWatchUploadElaspedTime.ElapsedMilliseconds, "ms"));
metrics.Add(new LongMetric(MetricName.maxTotalResults, 1, "#"));
metrics.Add(new LongMetric(MetricName.totalReadResults, 1, "#"));
metrics.Add(new LongMetric(MetricName.wrongResultsCount, 0, "#"));
var tcr = new TestUnitResult(metrics);
tcr.State = transferRequest;
return(tcr);
}
protected TestUnitResult Download(IAssetAccess enclosureAccess)
{
if (enclosureAccess == null)
{
return(null);
}
List <IMetric> metrics = new List <IMetric>();
metrics.Add(new StringMetric(MetricName.url, enclosureAccess.Uri.ToString(), ""));
int i = 0;
int taskId = Task.CurrentId.Value;
DateTimeOffset timeStart = DateTimeOffset.UtcNow;
metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime"));
long totalSize = Convert.ToInt64(enclosureAccess.TotalSize);
log.DebugFormat("[{1}] Method #{3} {2}: GET {0} ({4}) ...", enclosureAccess.Uri, taskId, enclosureAccess.AccessMethod, i + 1, BytesToString(totalSize));
var requests = enclosureAccess.GetDownloadRequests();
long[] respTime = new long[requests.Count()];
long totalByteCounter = 0;
TestUnitResultStatus tcrStatus = TestUnitResultStatus.Complete;
if (requests == null || requests.Count() == 0)
{
log.DebugFormat("[{1}] No download request for {0}. Skipping test unit.", enclosureAccess.Uri, taskId);
tcrStatus = TestUnitResultStatus.NotStarted;
}
Stopwatch stopWatchDownloadElaspedTime = new Stopwatch();
metrics.Add(new LongMetric(MetricName.beginGetResponseTime, DateTimeOffset.UtcNow.Ticks, "ticks"));
int j = 0;
foreach (var request in requests)
{
if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2)
{
// Test Breaker at 20MB )
break;
}
j++;
log.DebugFormat("[{1}] File #{0}/{3}: GET {2} ...", j, taskId, request.RequestUri, requests.Count());
Stopwatch sw = new Stopwatch();
sw.Start();
var downloadTask = request.GetResponseAsync().ContinueWith(resp =>
{
sw.Stop();
respTime[i] = sw.ElapsedMilliseconds;
using (ITransferResponse response = resp.Result)
{
log.InfoFormat("[{1}] > {3} Status Code {0} ({2}ms)", response.StatusCode, taskId, respTime[i], enclosureAccess.AccessMethod);
metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)response.StatusCode, response.StatusDescription), ""));
if (!(response.StatusCode == TransferStatusCode.OK || response.StatusCode == TransferStatusCode.Accepted))
{
log.DebugFormat("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription);
metrics.Add(new ExceptionMetric(new Exception(
string.Format("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription))));
}
else
{
using (var responseStream = response.GetResponseStream())
{
if (response.StatusCode == TransferStatusCode.OK && !response.ContentType.StartsWith("text/html"))
{
string hsize = "unknown";
if (response.ContentLength > 0)
{
hsize = BytesToString(response.ContentLength);
if (totalSize == 0)
{
totalSize = response.ContentLength;
}
}
log.InfoFormat("[{1}] < Content Type {2} Length {0}", hsize, taskId, response.ContentType);
byte[] buf = new byte[4096];
long fileSize = response.ContentLength;
double progessPct = 0;
long fileByteCounter = 0;
long intermediateCounter = 0;
stopWatchDownloadElaspedTime.Start();
int nbByteRead = responseStream.Read(buf, 0, 4096);
while (nbByteRead > 0)
{
totalByteCounter += nbByteRead;
fileByteCounter += nbByteRead;
intermediateCounter += nbByteRead;
double totalPct = ((double)totalByteCounter / totalSize) * 100;
double filePct = ((double)fileByteCounter / fileSize) * 100;
string hpct = "";
if ((fileSize > 0 && Math.Abs(progessPct - totalPct) > 1) || intermediateCounter >= 10485760)
{
if (fileSize > 0)
{
progessPct = totalPct;
hpct = string.Format(" (file #{2}:{1}% / total:{0}%)", totalPct.ToString("F1"), filePct.ToString("F1"), j);
}
double bytespersec = (double)totalByteCounter / ((double)stopWatchDownloadElaspedTime.ElapsedMilliseconds / 1000);
log.DebugFormat("[{3}] {0} downloaded{1} [{2}/s]", BytesToString(totalByteCounter), hpct, BytesToString((long)bytespersec), taskId);
intermediateCounter = 0;
}
nbByteRead = responseStream.Read(buf, 0, 4096);
if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2)
{
// Test Breaker at 20MB )
log.Warn("!!!!!TEST DOWNLOAD BREAKER!!!!");
break;
}
}
stopWatchDownloadElaspedTime.Stop();
}
}
}
}
});
try
{
downloadTask.Wait();
}
catch (Exception e)
{
Exception ie = e;
while (ie.InnerException != null)
{
ie = ie.InnerException;
}
if (ie is WebException)
{
WebException we = ie as WebException;
log.DebugFormat("[{0}] Error downloading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, enclosureAccess.Uri, we.Status.ToString(), enclosureAccess.AccessMethod);
if (we.Response is HttpWebResponse)
{
metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), ""));
}
metrics.Add(new ExceptionMetric(we));
}
else
{
log.DebugFormat("[{0}] Error during download {2}. Exception: {1}", Task.CurrentId, ie.Message, enclosureAccess.Uri);
log.Debug(ie.StackTrace);
metrics.Add(new ExceptionMetric(ie));
}
break;
}
i++;
}
DateTimeOffset timeStop = DateTimeOffset.UtcNow;
metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime"));
metrics.Add(new LongMetric(MetricName.endGetResponseTime, DateTime.UtcNow.Ticks, "ticks"));
if (respTime.Count() > 0)
{
metrics.Add(new LongMetric(MetricName.responseTime, Convert.ToInt64(respTime.Average()), "ms"));
}
else
{
metrics.Add(new LongMetric(MetricName.responseTime, 0, "ms"));
}
metrics.Add(new LongMetric(MetricName.size, totalByteCounter, "bytes"));
metrics.Add(new LongMetric(MetricName.downloadElapsedTime, stopWatchDownloadElaspedTime.ElapsedMilliseconds, "ms"));
metrics.Add(new LongMetric(MetricName.maxTotalResults, 1, "#"));
metrics.Add(new LongMetric(MetricName.totalReadResults, 1, "#"));
metrics.Add(new LongMetric(MetricName.wrongResultsCount, 0, "#"));
var tcr = new TestUnitResult(metrics, tcrStatus);
if (enclosureAccess.SourceItem != null)
{
FiltersDefinition fd = DataHelper.GenerateFiltersDefinitionFromItem("Download", enclosureAccess.SourceItem);
metrics.Add(new StringMetric(MetricName.dataCollectionDivision, fd.Label, "string"));
tcr.FiltersDefinition = fd;
}
tcr.State = enclosureAccess;
return(tcr);
}