protected virtual void CreateDownloadRequestAndEnqueue(IOpenSearchResultItem item)
{
log.DebugFormat("Creating download request for {0}...", item.Identifier);
try
{
IAssetAccess enclosureAccess = target.Wrapper.GetEnclosureAccess(item);
if (enclosureAccess != null)
{
if (Convert.ToInt64(enclosureAccess.TotalSize) > target.TargetSiteConfig.MaxDownloadSize)
{
throw new Exception(String.Format("Product file too large ({0} > {1}) for {2}", enclosureAccess.TotalSize, target.TargetSiteConfig.MaxDownloadSize, item.Identifier));
}
downloadRequests.Enqueue(enclosureAccess);
log.DebugFormat("OK --> {0}", enclosureAccess.Uri);
}
}
catch (Exception e)
{
log.WarnFormat("NOT OK: {0}", e.Message);
log.Debug(e.StackTrace);
}
}
public override IEnumerable <TestUnitResult> RunTestUnits(TaskFactory taskFactory, Task prepTask)
{
StartTime = DateTimeOffset.UtcNow;
List <Task <TestUnitResult> > _testUnits = new List <Task <TestUnitResult> >();
Task[] previousTask = Array.ConvertAll(new int[target.TargetSiteConfig.MaxDownloadThread], i => prepTask);
if (!downloadRequests.IsEmpty)
{
log.InfoFormat("Starting concurrent download(s) [{0}] of {1} item(s)", target.TargetSiteConfig.MaxDownloadThread, load_factor);
}
for (int i = 0; i < load_factor && !downloadRequests.IsEmpty;)
{
for (int j = 0; j < load_factor && j < target.TargetSiteConfig.MaxDownloadThread && !downloadRequests.IsEmpty; j++, i++)
{
Task <TestUnitResult> _testUnit = previousTask[j].ContinueWith <IAssetAccess>((task) =>
{
prepTask.Wait();
IAssetAccess enclosureAccess = null;
if (downloadRequests.TryDequeue(out enclosureAccess))
{
log.DebugFormat("Dequeuing {0}", enclosureAccess.Uri);
return(enclosureAccess);
}
return(null);
}).ContinueWith((request) => Download(request.Result));
_testUnits.Add(_testUnit);
previousTask[j] = _testUnit;
}
}
Task.WaitAll(_testUnits.ToArray());
EndTime = DateTimeOffset.UtcNow;
return(_testUnits.Select(t => t.Result));
}
protected TestUnitResult Download(IAssetAccess enclosureAccess)
{
if (enclosureAccess == null)
{
return(null);
}
List <IMetric> metrics = new List <IMetric>();
metrics.Add(new StringMetric(MetricName.url, enclosureAccess.Uri.ToString(), ""));
int i = 0;
int taskId = Task.CurrentId.Value;
DateTimeOffset timeStart = DateTimeOffset.UtcNow;
metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime"));
long totalSize = Convert.ToInt64(enclosureAccess.TotalSize);
log.DebugFormat("[{1}] Method #{3} {2}: GET {0} ({4}) ...", enclosureAccess.Uri, taskId, enclosureAccess.AccessMethod, i + 1, BytesToString(totalSize));
var requests = enclosureAccess.GetDownloadRequests();
long[] respTime = new long[requests.Count()];
long totalByteCounter = 0;
TestUnitResultStatus tcrStatus = TestUnitResultStatus.Complete;
if (requests == null || requests.Count() == 0)
{
log.DebugFormat("[{1}] No download request for {0}. Skipping test unit.", enclosureAccess.Uri, taskId);
tcrStatus = TestUnitResultStatus.NotStarted;
}
Stopwatch stopWatchDownloadElaspedTime = new Stopwatch();
metrics.Add(new LongMetric(MetricName.beginGetResponseTime, DateTimeOffset.UtcNow.Ticks, "ticks"));
int j = 0;
foreach (var request in requests)
{
if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2)
{
// Test Breaker at 20MB )
break;
}
j++;
log.DebugFormat("[{1}] File #{0}/{3}: GET {2} ...", j, taskId, request.RequestUri, requests.Count());
Stopwatch sw = new Stopwatch();
sw.Start();
var downloadTask = request.GetResponseAsync().ContinueWith(resp =>
{
sw.Stop();
respTime[i] = sw.ElapsedMilliseconds;
using (ITransferResponse response = resp.Result)
{
log.InfoFormat("[{1}] > {3} Status Code {0} ({2}ms)", response.StatusCode, taskId, respTime[i], enclosureAccess.AccessMethod);
metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)response.StatusCode, response.StatusDescription), ""));
if (!(response.StatusCode == TransferStatusCode.OK || response.StatusCode == TransferStatusCode.Accepted))
{
log.DebugFormat("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription);
metrics.Add(new ExceptionMetric(new Exception(
string.Format("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription))));
}
else
{
using (var responseStream = response.GetResponseStream())
{
if (response.StatusCode == TransferStatusCode.OK && !response.ContentType.StartsWith("text/html"))
{
string hsize = "unknown";
if (response.ContentLength > 0)
{
hsize = BytesToString(response.ContentLength);
if (totalSize == 0)
{
totalSize = response.ContentLength;
}
}
log.InfoFormat("[{1}] < Content Type {2} Length {0}", hsize, taskId, response.ContentType);
byte[] buf = new byte[4096];
long fileSize = response.ContentLength;
double progessPct = 0;
long fileByteCounter = 0;
long intermediateCounter = 0;
stopWatchDownloadElaspedTime.Start();
int nbByteRead = responseStream.Read(buf, 0, 4096);
while (nbByteRead > 0)
{
totalByteCounter += nbByteRead;
fileByteCounter += nbByteRead;
intermediateCounter += nbByteRead;
double totalPct = ((double)totalByteCounter / totalSize) * 100;
double filePct = ((double)fileByteCounter / fileSize) * 100;
string hpct = "";
if ((fileSize > 0 && Math.Abs(progessPct - totalPct) > 1) || intermediateCounter >= 10485760)
{
if (fileSize > 0)
{
progessPct = totalPct;
hpct = string.Format(" (file #{2}:{1}% / total:{0}%)", totalPct.ToString("F1"), filePct.ToString("F1"), j);
}
double bytespersec = (double)totalByteCounter / ((double)stopWatchDownloadElaspedTime.ElapsedMilliseconds / 1000);
log.DebugFormat("[{3}] {0} downloaded{1} [{2}/s]", BytesToString(totalByteCounter), hpct, BytesToString((long)bytespersec), taskId);
intermediateCounter = 0;
}
nbByteRead = responseStream.Read(buf, 0, 4096);
if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2)
{
// Test Breaker at 20MB )
log.Warn("!!!!!TEST DOWNLOAD BREAKER!!!!");
break;
}
}
stopWatchDownloadElaspedTime.Stop();
}
}
}
}
});
try
{
downloadTask.Wait();
}
catch (Exception e)
{
Exception ie = e;
while (ie.InnerException != null)
{
ie = ie.InnerException;
}
if (ie is WebException)
{
WebException we = ie as WebException;
log.DebugFormat("[{0}] Error downloading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, enclosureAccess.Uri, we.Status.ToString(), enclosureAccess.AccessMethod);
if (we.Response is HttpWebResponse)
{
metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), ""));
}
metrics.Add(new ExceptionMetric(we));
}
else
{
log.DebugFormat("[{0}] Error during download {2}. Exception: {1}", Task.CurrentId, ie.Message, enclosureAccess.Uri);
log.Debug(ie.StackTrace);
metrics.Add(new ExceptionMetric(ie));
}
break;
}
i++;
}
DateTimeOffset timeStop = DateTimeOffset.UtcNow;
metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime"));
metrics.Add(new LongMetric(MetricName.endGetResponseTime, DateTime.UtcNow.Ticks, "ticks"));
if (respTime.Count() > 0)
{
metrics.Add(new LongMetric(MetricName.responseTime, Convert.ToInt64(respTime.Average()), "ms"));
}
else
{
metrics.Add(new LongMetric(MetricName.responseTime, 0, "ms"));
}
metrics.Add(new LongMetric(MetricName.size, totalByteCounter, "bytes"));
metrics.Add(new LongMetric(MetricName.downloadElapsedTime, stopWatchDownloadElaspedTime.ElapsedMilliseconds, "ms"));
metrics.Add(new LongMetric(MetricName.maxTotalResults, 1, "#"));
metrics.Add(new LongMetric(MetricName.totalReadResults, 1, "#"));
metrics.Add(new LongMetric(MetricName.wrongResultsCount, 0, "#"));
var tcr = new TestUnitResult(metrics, tcrStatus);
if (enclosureAccess.SourceItem != null)
{
FiltersDefinition fd = DataHelper.GenerateFiltersDefinitionFromItem("Download", enclosureAccess.SourceItem);
metrics.Add(new StringMetric(MetricName.dataCollectionDivision, fd.Label, "string"));
tcr.FiltersDefinition = fd;
}
tcr.State = enclosureAccess;
return(tcr);
}