Beispiel #1
0
        protected TestUnitResult Upload(ITransferRequest transferRequest)
        {
            if (transferRequest == null)
            {
                return(null);
            }

            List <IMetric> metrics = new List <IMetric>();

            metrics.Add(new StringMetric(MetricName.url, transferRequest.RequestUri.ToString(), ""));

            int taskId = Task.CurrentId.Value;

            DateTimeOffset timeStart = DateTimeOffset.UtcNow;

            metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime"));

            List <ItemAccessMethod> methods = new List <ItemAccessMethod>()
            {
                transferRequest.Method
            };

            if (methods.Count() == 0)
            {
                log.WarnFormat("No upload methods found. Skipping upload.");
                var tcr1 = new TestUnitResult(metrics, TestUnitResultStatus.NotStarted);
                tcr1.State = transferRequest;
                metrics.Add(new DateTimeMetric(MetricName.endTime, DateTimeOffset.UtcNow, "dateTime"));
                return(tcr1);
            }

            var method = methods.First();

            long   respTime  = 0;
            Random rng       = new Random();
            long   totalSize = Convert.ToInt64(transferRequest.ContentLength);

            log.DebugFormat("Upload {1} to {0} ({2}) ...", transferRequest.RequestUri, BytesToString(Convert.ToInt64(totalSize)), transferRequest.Method);

            Stopwatch stopWatchUploadElaspedTime = new Stopwatch();

            metrics.Add(new LongMetric(MetricName.beginGetResponseTime, DateTimeOffset.UtcNow.Ticks, "ticks"));
            Stopwatch sw = new Stopwatch();

            sw.Start();

            var uploadStreamTask = transferRequest.GetRequestStreamAsync();

            uploadStreamTask.ContinueWith(streamTask =>
            {
                respTime                 = sw.ElapsedMilliseconds;
                Stream stream            = streamTask.Result;
                byte[] buf               = new byte[4096];
                double progessPct        = 0;
                long totalByteCounter    = 0;
                long intermediateCounter = 0;
                stopWatchUploadElaspedTime.Start();
                while (totalByteCounter < totalSize)
                {
                    rng.NextBytes(buf);
                    int length = buf.Length;
                    if (totalSize < totalByteCounter + length)
                    {
                        length = Convert.ToInt32(totalSize - totalByteCounter);
                    }
                    stream.Write(buf, 0, buf.Length);
                    totalByteCounter    += buf.Length;
                    intermediateCounter += buf.Length;
                    double totalPct      = ((double)totalByteCounter / totalSize) * 100;
                    if ((Math.Abs(progessPct - totalPct) >= 1) || intermediateCounter >= 10485760)
                    {
                        progessPct         = totalPct;
                        double bytespersec = (double)totalByteCounter / ((double)stopWatchUploadElaspedTime.ElapsedMilliseconds / 1000);
                        log.DebugFormat("[{2}] Uploaded {0}% [{1}/s]", totalPct.ToString("F1"), BytesToString((long)bytespersec), taskId);

                        intermediateCounter = 0;
                    }

                    if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2)
                    {
                        // Test Breaker at 20MB )
                        log.Warn("!!!!!TEST DOWNLOAD BREAKER!!!!");
                        break;
                    }
                }
                stream.Close();
                stopWatchUploadElaspedTime.Stop();
            });

            try
            {
                uploadStreamTask.Wait();
            }
            catch (Exception e)
            {
                Exception ie = e;
                while (ie.InnerException != null)
                {
                    ie = ie.InnerException;
                }
                if (ie is WebException)
                {
                    WebException we = ie as WebException;
                    log.DebugFormat("[{0}] Error uploading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, transferRequest.RequestUri, we.Status.ToString(), method);
                    if (we.Response is HttpWebResponse)
                    {
                        metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), ""));
                    }

                    metrics.Add(new ExceptionMetric(we));
                }
                else
                {
                    log.DebugFormat("[{0}] Error during upload {2}. Exception: {1}", Task.CurrentId, ie.Message, transferRequest.RequestUri);
                    log.Debug(ie.StackTrace);
                    metrics.Add(new ExceptionMetric(ie));
                }
            }

            var uploadTask = transferRequest.GetResponseAsync().ContinueWith(resp =>
            {
                sw.Stop();

                using (ITransferResponse response = resp.Result)
                {
                    log.InfoFormat("[{1}] > {3} Status Code {0} ({2}ms)", response.StatusCode, taskId, respTime, method);
                    metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)response.StatusCode, response.StatusDescription), ""));
                    if (!(response.StatusCode == TransferStatusCode.OK || response.StatusCode == TransferStatusCode.Accepted || response.StatusCode == TransferStatusCode.Created))
                    {
                        log.DebugFormat("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription);
                        metrics.Add(new ExceptionMetric(new Exception(
                                                            string.Format("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription))));
                    }
                    else
                    {
                        log.InfoFormat("[{0}] < OK. Content Type {1}", taskId, response.ContentType);
                    }
                }
            });

            try
            {
                uploadTask.Wait();
            }
            catch (Exception e)
            {
                Exception ie = e;
                while (ie.InnerException != null)
                {
                    ie = ie.InnerException;
                }
                if (ie is WebException)
                {
                    WebException we = ie as WebException;
                    log.DebugFormat("[{0}] Error uploading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, transferRequest.RequestUri, we.Status.ToString(), method);
                    if (we.Response is HttpWebResponse)
                    {
                        metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), ""));
                    }

                    metrics.Add(new ExceptionMetric(we));
                }
                else
                {
                    log.DebugFormat("[{0}] Error during upload {2}. Exception: {1}", Task.CurrentId, ie.Message, transferRequest.RequestUri);
                    log.Debug(ie.StackTrace);
                    metrics.Add(new ExceptionMetric(ie));
                }
            }

            DateTimeOffset timeStop = DateTimeOffset.UtcNow;

            metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime"));
            metrics.Add(new LongMetric(MetricName.endGetResponseTime, DateTime.UtcNow.Ticks, "ticks"));
            metrics.Add(new LongMetric(MetricName.responseTime, respTime, "ms"));
            metrics.Add(new LongMetric(MetricName.size, totalSize, "bytes"));
            metrics.Add(new LongMetric(MetricName.downloadElapsedTime, stopWatchUploadElaspedTime.ElapsedMilliseconds, "ms"));
            metrics.Add(new LongMetric(MetricName.maxTotalResults, 1, "#"));
            metrics.Add(new LongMetric(MetricName.totalReadResults, 1, "#"));
            metrics.Add(new LongMetric(MetricName.wrongResultsCount, 0, "#"));
            var tcr = new TestUnitResult(metrics);

            tcr.State = transferRequest;
            return(tcr);
        }
Beispiel #2
0
        protected TestUnitResult Download(IAssetAccess enclosureAccess)
        {
            if (enclosureAccess == null)
            {
                return(null);
            }

            List <IMetric> metrics = new List <IMetric>();

            metrics.Add(new StringMetric(MetricName.url, enclosureAccess.Uri.ToString(), ""));

            int i      = 0;
            int taskId = Task.CurrentId.Value;

            DateTimeOffset timeStart = DateTimeOffset.UtcNow;

            metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime"));

            long totalSize = Convert.ToInt64(enclosureAccess.TotalSize);

            log.DebugFormat("[{1}] Method #{3} {2}: GET {0} ({4}) ...", enclosureAccess.Uri, taskId, enclosureAccess.AccessMethod, i + 1, BytesToString(totalSize));
            var requests = enclosureAccess.GetDownloadRequests();

            long[] respTime         = new long[requests.Count()];
            long   totalByteCounter = 0;

            TestUnitResultStatus tcrStatus = TestUnitResultStatus.Complete;

            if (requests == null || requests.Count() == 0)
            {
                log.DebugFormat("[{1}] No download request for {0}. Skipping test unit.", enclosureAccess.Uri, taskId);
                tcrStatus = TestUnitResultStatus.NotStarted;
            }

            Stopwatch stopWatchDownloadElaspedTime = new Stopwatch();

            metrics.Add(new LongMetric(MetricName.beginGetResponseTime, DateTimeOffset.UtcNow.Ticks, "ticks"));
            int j = 0;

            foreach (var request in requests)
            {
                if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2)
                {
                    // Test Breaker at 20MB )
                    break;
                }
                j++;
                log.DebugFormat("[{1}] File #{0}/{3}: GET {2} ...", j, taskId, request.RequestUri, requests.Count());
                Stopwatch sw = new Stopwatch();
                sw.Start();

                var downloadTask = request.GetResponseAsync().ContinueWith(resp =>
                {
                    sw.Stop();

                    respTime[i] = sw.ElapsedMilliseconds;

                    using (ITransferResponse response = resp.Result)
                    {
                        log.InfoFormat("[{1}] > {3} Status Code {0} ({2}ms)", response.StatusCode, taskId, respTime[i], enclosureAccess.AccessMethod);
                        metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)response.StatusCode, response.StatusDescription), ""));
                        if (!(response.StatusCode == TransferStatusCode.OK || response.StatusCode == TransferStatusCode.Accepted))
                        {
                            log.DebugFormat("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription);
                            metrics.Add(new ExceptionMetric(new Exception(
                                                                string.Format("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription))));
                        }
                        else
                        {
                            using (var responseStream = response.GetResponseStream())
                            {
                                if (response.StatusCode == TransferStatusCode.OK && !response.ContentType.StartsWith("text/html"))
                                {
                                    string hsize = "unknown";
                                    if (response.ContentLength > 0)
                                    {
                                        hsize = BytesToString(response.ContentLength);
                                        if (totalSize == 0)
                                        {
                                            totalSize = response.ContentLength;
                                        }
                                    }
                                    log.InfoFormat("[{1}] < Content Type {2} Length {0}", hsize, taskId, response.ContentType);
                                    byte[] buf               = new byte[4096];
                                    long fileSize            = response.ContentLength;
                                    double progessPct        = 0;
                                    long fileByteCounter     = 0;
                                    long intermediateCounter = 0;
                                    stopWatchDownloadElaspedTime.Start();
                                    int nbByteRead = responseStream.Read(buf, 0, 4096);
                                    while (nbByteRead > 0)
                                    {
                                        totalByteCounter    += nbByteRead;
                                        fileByteCounter     += nbByteRead;
                                        intermediateCounter += nbByteRead;
                                        double totalPct      = ((double)totalByteCounter / totalSize) * 100;
                                        double filePct       = ((double)fileByteCounter / fileSize) * 100;
                                        string hpct          = "";
                                        if ((fileSize > 0 && Math.Abs(progessPct - totalPct) > 1) || intermediateCounter >= 10485760)
                                        {
                                            if (fileSize > 0)
                                            {
                                                progessPct = totalPct;
                                                hpct       = string.Format(" (file #{2}:{1}% / total:{0}%)", totalPct.ToString("F1"), filePct.ToString("F1"), j);
                                            }
                                            double bytespersec = (double)totalByteCounter / ((double)stopWatchDownloadElaspedTime.ElapsedMilliseconds / 1000);
                                            log.DebugFormat("[{3}] {0} downloaded{1} [{2}/s]", BytesToString(totalByteCounter), hpct, BytesToString((long)bytespersec), taskId);
                                            intermediateCounter = 0;
                                        }
                                        nbByteRead = responseStream.Read(buf, 0, 4096);

                                        if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2)
                                        {
                                            // Test Breaker at 20MB )
                                            log.Warn("!!!!!TEST DOWNLOAD BREAKER!!!!");
                                            break;
                                        }
                                    }
                                    stopWatchDownloadElaspedTime.Stop();
                                }
                            }
                        }
                    }
                });
                try
                {
                    downloadTask.Wait();
                }
                catch (Exception e)
                {
                    Exception ie = e;
                    while (ie.InnerException != null)
                    {
                        ie = ie.InnerException;
                    }
                    if (ie is WebException)
                    {
                        WebException we = ie as WebException;
                        log.DebugFormat("[{0}] Error downloading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, enclosureAccess.Uri, we.Status.ToString(), enclosureAccess.AccessMethod);
                        if (we.Response is HttpWebResponse)
                        {
                            metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), ""));
                        }

                        metrics.Add(new ExceptionMetric(we));
                    }
                    else
                    {
                        log.DebugFormat("[{0}] Error during download {2}. Exception: {1}", Task.CurrentId, ie.Message, enclosureAccess.Uri);
                        log.Debug(ie.StackTrace);
                        metrics.Add(new ExceptionMetric(ie));
                    }
                    break;
                }

                i++;
            }
            DateTimeOffset timeStop = DateTimeOffset.UtcNow;

            metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime"));
            metrics.Add(new LongMetric(MetricName.endGetResponseTime, DateTime.UtcNow.Ticks, "ticks"));
            if (respTime.Count() > 0)
            {
                metrics.Add(new LongMetric(MetricName.responseTime, Convert.ToInt64(respTime.Average()), "ms"));
            }
            else
            {
                metrics.Add(new LongMetric(MetricName.responseTime, 0, "ms"));
            }
            metrics.Add(new LongMetric(MetricName.size, totalByteCounter, "bytes"));
            metrics.Add(new LongMetric(MetricName.downloadElapsedTime, stopWatchDownloadElaspedTime.ElapsedMilliseconds, "ms"));
            metrics.Add(new LongMetric(MetricName.maxTotalResults, 1, "#"));
            metrics.Add(new LongMetric(MetricName.totalReadResults, 1, "#"));
            metrics.Add(new LongMetric(MetricName.wrongResultsCount, 0, "#"));
            var tcr = new TestUnitResult(metrics, tcrStatus);

            if (enclosureAccess.SourceItem != null)
            {
                FiltersDefinition fd = DataHelper.GenerateFiltersDefinitionFromItem("Download", enclosureAccess.SourceItem);
                metrics.Add(new StringMetric(MetricName.dataCollectionDivision, fd.Label, "string"));
                tcr.FiltersDefinition = fd;
            }
            tcr.State = enclosureAccess;
            return(tcr);
        }