protected TestUnitResult Upload(ITransferRequest transferRequest) { if (transferRequest == null) { return(null); } List <IMetric> metrics = new List <IMetric>(); metrics.Add(new StringMetric(MetricName.url, transferRequest.RequestUri.ToString(), "")); int taskId = Task.CurrentId.Value; DateTimeOffset timeStart = DateTimeOffset.UtcNow; metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime")); List <ItemAccessMethod> methods = new List <ItemAccessMethod>() { transferRequest.Method }; if (methods.Count() == 0) { log.WarnFormat("No upload methods found. Skipping upload."); var tcr1 = new TestUnitResult(metrics, TestUnitResultStatus.NotStarted); tcr1.State = transferRequest; metrics.Add(new DateTimeMetric(MetricName.endTime, DateTimeOffset.UtcNow, "dateTime")); return(tcr1); } var method = methods.First(); long respTime = 0; Random rng = new Random(); long totalSize = Convert.ToInt64(transferRequest.ContentLength); log.DebugFormat("Upload {1} to {0} ({2}) ...", transferRequest.RequestUri, BytesToString(Convert.ToInt64(totalSize)), transferRequest.Method); Stopwatch stopWatchUploadElaspedTime = new Stopwatch(); metrics.Add(new LongMetric(MetricName.beginGetResponseTime, DateTimeOffset.UtcNow.Ticks, "ticks")); Stopwatch sw = new Stopwatch(); sw.Start(); var uploadStreamTask = transferRequest.GetRequestStreamAsync(); uploadStreamTask.ContinueWith(streamTask => { respTime = sw.ElapsedMilliseconds; Stream stream = streamTask.Result; byte[] buf = new byte[4096]; double progessPct = 0; long totalByteCounter = 0; long intermediateCounter = 0; stopWatchUploadElaspedTime.Start(); while (totalByteCounter < totalSize) { rng.NextBytes(buf); int length = buf.Length; if (totalSize < totalByteCounter + length) { length = Convert.ToInt32(totalSize - totalByteCounter); } stream.Write(buf, 0, buf.Length); totalByteCounter += buf.Length; intermediateCounter += buf.Length; double totalPct = ((double)totalByteCounter / totalSize) * 100; if ((Math.Abs(progessPct - totalPct) >= 1) || intermediateCounter >= 10485760) { progessPct = totalPct; double bytespersec = (double)totalByteCounter / ((double)stopWatchUploadElaspedTime.ElapsedMilliseconds / 1000); log.DebugFormat("[{2}] Uploaded {0}% [{1}/s]", totalPct.ToString("F1"), BytesToString((long)bytespersec), taskId); intermediateCounter = 0; } if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2) { // Test Breaker at 20MB ) log.Warn("!!!!!TEST DOWNLOAD BREAKER!!!!"); break; } } stream.Close(); stopWatchUploadElaspedTime.Stop(); }); try { uploadStreamTask.Wait(); } catch (Exception e) { Exception ie = e; while (ie.InnerException != null) { ie = ie.InnerException; } if (ie is WebException) { WebException we = ie as WebException; log.DebugFormat("[{0}] Error uploading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, transferRequest.RequestUri, we.Status.ToString(), method); if (we.Response is HttpWebResponse) { metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), "")); } metrics.Add(new ExceptionMetric(we)); } else { log.DebugFormat("[{0}] Error during upload {2}. Exception: {1}", Task.CurrentId, ie.Message, transferRequest.RequestUri); log.Debug(ie.StackTrace); metrics.Add(new ExceptionMetric(ie)); } } var uploadTask = transferRequest.GetResponseAsync().ContinueWith(resp => { sw.Stop(); using (ITransferResponse response = resp.Result) { log.InfoFormat("[{1}] > {3} Status Code {0} ({2}ms)", response.StatusCode, taskId, respTime, method); metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)response.StatusCode, response.StatusDescription), "")); if (!(response.StatusCode == TransferStatusCode.OK || response.StatusCode == TransferStatusCode.Accepted || response.StatusCode == TransferStatusCode.Created)) { log.DebugFormat("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription); metrics.Add(new ExceptionMetric(new Exception( string.Format("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription)))); } else { log.InfoFormat("[{0}] < OK. Content Type {1}", taskId, response.ContentType); } } }); try { uploadTask.Wait(); } catch (Exception e) { Exception ie = e; while (ie.InnerException != null) { ie = ie.InnerException; } if (ie is WebException) { WebException we = ie as WebException; log.DebugFormat("[{0}] Error uploading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, transferRequest.RequestUri, we.Status.ToString(), method); if (we.Response is HttpWebResponse) { metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), "")); } metrics.Add(new ExceptionMetric(we)); } else { log.DebugFormat("[{0}] Error during upload {2}. Exception: {1}", Task.CurrentId, ie.Message, transferRequest.RequestUri); log.Debug(ie.StackTrace); metrics.Add(new ExceptionMetric(ie)); } } DateTimeOffset timeStop = DateTimeOffset.UtcNow; metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime")); metrics.Add(new LongMetric(MetricName.endGetResponseTime, DateTime.UtcNow.Ticks, "ticks")); metrics.Add(new LongMetric(MetricName.responseTime, respTime, "ms")); metrics.Add(new LongMetric(MetricName.size, totalSize, "bytes")); metrics.Add(new LongMetric(MetricName.downloadElapsedTime, stopWatchUploadElaspedTime.ElapsedMilliseconds, "ms")); metrics.Add(new LongMetric(MetricName.maxTotalResults, 1, "#")); metrics.Add(new LongMetric(MetricName.totalReadResults, 1, "#")); metrics.Add(new LongMetric(MetricName.wrongResultsCount, 0, "#")); var tcr = new TestUnitResult(metrics); tcr.State = transferRequest; return(tcr); }
protected TestUnitResult Download(IAssetAccess enclosureAccess) { if (enclosureAccess == null) { return(null); } List <IMetric> metrics = new List <IMetric>(); metrics.Add(new StringMetric(MetricName.url, enclosureAccess.Uri.ToString(), "")); int i = 0; int taskId = Task.CurrentId.Value; DateTimeOffset timeStart = DateTimeOffset.UtcNow; metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime")); long totalSize = Convert.ToInt64(enclosureAccess.TotalSize); log.DebugFormat("[{1}] Method #{3} {2}: GET {0} ({4}) ...", enclosureAccess.Uri, taskId, enclosureAccess.AccessMethod, i + 1, BytesToString(totalSize)); var requests = enclosureAccess.GetDownloadRequests(); long[] respTime = new long[requests.Count()]; long totalByteCounter = 0; TestUnitResultStatus tcrStatus = TestUnitResultStatus.Complete; if (requests == null || requests.Count() == 0) { log.DebugFormat("[{1}] No download request for {0}. Skipping test unit.", enclosureAccess.Uri, taskId); tcrStatus = TestUnitResultStatus.NotStarted; } Stopwatch stopWatchDownloadElaspedTime = new Stopwatch(); metrics.Add(new LongMetric(MetricName.beginGetResponseTime, DateTimeOffset.UtcNow.Ticks, "ticks")); int j = 0; foreach (var request in requests) { if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2) { // Test Breaker at 20MB ) break; } j++; log.DebugFormat("[{1}] File #{0}/{3}: GET {2} ...", j, taskId, request.RequestUri, requests.Count()); Stopwatch sw = new Stopwatch(); sw.Start(); var downloadTask = request.GetResponseAsync().ContinueWith(resp => { sw.Stop(); respTime[i] = sw.ElapsedMilliseconds; using (ITransferResponse response = resp.Result) { log.InfoFormat("[{1}] > {3} Status Code {0} ({2}ms)", response.StatusCode, taskId, respTime[i], enclosureAccess.AccessMethod); metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)response.StatusCode, response.StatusDescription), "")); if (!(response.StatusCode == TransferStatusCode.OK || response.StatusCode == TransferStatusCode.Accepted)) { log.DebugFormat("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription); metrics.Add(new ExceptionMetric(new Exception( string.Format("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription)))); } else { using (var responseStream = response.GetResponseStream()) { if (response.StatusCode == TransferStatusCode.OK && !response.ContentType.StartsWith("text/html")) { string hsize = "unknown"; if (response.ContentLength > 0) { hsize = BytesToString(response.ContentLength); if (totalSize == 0) { totalSize = response.ContentLength; } } log.InfoFormat("[{1}] < Content Type {2} Length {0}", hsize, taskId, response.ContentType); byte[] buf = new byte[4096]; long fileSize = response.ContentLength; double progessPct = 0; long fileByteCounter = 0; long intermediateCounter = 0; stopWatchDownloadElaspedTime.Start(); int nbByteRead = responseStream.Read(buf, 0, 4096); while (nbByteRead > 0) { totalByteCounter += nbByteRead; fileByteCounter += nbByteRead; intermediateCounter += nbByteRead; double totalPct = ((double)totalByteCounter / totalSize) * 100; double filePct = ((double)fileByteCounter / fileSize) * 100; string hpct = ""; if ((fileSize > 0 && Math.Abs(progessPct - totalPct) > 1) || intermediateCounter >= 10485760) { if (fileSize > 0) { progessPct = totalPct; hpct = string.Format(" (file #{2}:{1}% / total:{0}%)", totalPct.ToString("F1"), filePct.ToString("F1"), j); } double bytespersec = (double)totalByteCounter / ((double)stopWatchDownloadElaspedTime.ElapsedMilliseconds / 1000); log.DebugFormat("[{3}] {0} downloaded{1} [{2}/s]", BytesToString(totalByteCounter), hpct, BytesToString((long)bytespersec), taskId); intermediateCounter = 0; } nbByteRead = responseStream.Read(buf, 0, 4096); if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2) { // Test Breaker at 20MB ) log.Warn("!!!!!TEST DOWNLOAD BREAKER!!!!"); break; } } stopWatchDownloadElaspedTime.Stop(); } } } } }); try { downloadTask.Wait(); } catch (Exception e) { Exception ie = e; while (ie.InnerException != null) { ie = ie.InnerException; } if (ie is WebException) { WebException we = ie as WebException; log.DebugFormat("[{0}] Error downloading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, enclosureAccess.Uri, we.Status.ToString(), enclosureAccess.AccessMethod); if (we.Response is HttpWebResponse) { metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), "")); } metrics.Add(new ExceptionMetric(we)); } else { log.DebugFormat("[{0}] Error during download {2}. Exception: {1}", Task.CurrentId, ie.Message, enclosureAccess.Uri); log.Debug(ie.StackTrace); metrics.Add(new ExceptionMetric(ie)); } break; } i++; } DateTimeOffset timeStop = DateTimeOffset.UtcNow; metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime")); metrics.Add(new LongMetric(MetricName.endGetResponseTime, DateTime.UtcNow.Ticks, "ticks")); if (respTime.Count() > 0) { metrics.Add(new LongMetric(MetricName.responseTime, Convert.ToInt64(respTime.Average()), "ms")); } else { metrics.Add(new LongMetric(MetricName.responseTime, 0, "ms")); } metrics.Add(new LongMetric(MetricName.size, totalByteCounter, "bytes")); metrics.Add(new LongMetric(MetricName.downloadElapsedTime, stopWatchDownloadElaspedTime.ElapsedMilliseconds, "ms")); metrics.Add(new LongMetric(MetricName.maxTotalResults, 1, "#")); metrics.Add(new LongMetric(MetricName.totalReadResults, 1, "#")); metrics.Add(new LongMetric(MetricName.wrongResultsCount, 0, "#")); var tcr = new TestUnitResult(metrics, tcrStatus); if (enclosureAccess.SourceItem != null) { FiltersDefinition fd = DataHelper.GenerateFiltersDefinitionFromItem("Download", enclosureAccess.SourceItem); metrics.Add(new StringMetric(MetricName.dataCollectionDivision, fd.Label, "string")); tcr.FiltersDefinition = fd; } tcr.State = enclosureAccess; return(tcr); }