private void DownloadResource_(string url, List <Task <RGridResource> > tasks)
{
url = SanitizeUrl_(url);
tasks.Add(Task.Run(async() =>
{
if (!fetchedCacheMap_.TryGetValue(url, out ResourceFuture future))
{
future = DownloadResourceFuture_(url);
if (fetchedCacheMap_.TryAdd(url, future))
{
logger_.Verbose("fetchedCacheMap_.Add({0})", url);
}
}
RGridResource r = await future.Get(TimeSpan.FromSeconds(60));
debugResourceWriter_.Write(r);
HashSet <string> newResources = new HashSet <string>();
GetAndParseResource_(r, r.Url, newResources);
string sanitizedUrl = SanitizeUrl_(r.Url.OriginalString);
cachedResourceMapping_.TryAdd(sanitizedUrl, newResources);
logger_.Verbose("adding {0} more resources to download queue.", newResources.Count);
foreach (string rUrl in newResources)
{
sanitizedUrl = SanitizeUrl_(rUrl);
if (fetchedCacheMap_.ContainsKey(sanitizedUrl))
{
continue;
}
resourcesToFetch_.Enqueue(sanitizedUrl);
logger_.Verbose("added url to download: {0}", sanitizedUrl);
}
return(r);
}));
}
public PutFuture(RGridResource resource, RunningRender runningRender, IEyesConnector eyesConnector, Logger logger)
{
resource_ = resource;
runningRender_ = runningRender;
eyesConnector_ = eyesConnector;
logger_ = logger;
}
internal void FetchAllResources_(IDictionary <string, RGridResource> allBlobs, HashSet <string> resourceUrls)
{
logger_.Verbose("enter");
resourcesToFetch_ = new Queue <string>(resourceUrls);
while (resourcesToFetch_.Count > 0)
{
var tasks = new List <Task <RGridResource> >();
while (resourcesToFetch_.Count > 0)
{
string url = resourcesToFetch_.Dequeue();
DownloadResource_(url, tasks);
}
while (tasks.Any(t => !t.IsCompleted))
{
Task.WaitAll(tasks.ToArray());
}
foreach (Task <RGridResource> task in tasks)
{
RGridResource resource = task.Result;
string url = resource.Url.OriginalString;
if (!allBlobs.ContainsKey(url))
{
allBlobs.Add(url, resource);
}
}
}
logger_.Verbose("exit");
}
public async Task <RGridResource> Get(TimeSpan timeout)
{
if (rgResource_ == null)
{
logger_.Debug("{0} Future ref#: {1}", Url, Future.GetHashCode());
if (await Task.WhenAny(Future, Task.Delay(timeout)) == Future)
{
if (Future.Exception == null)
{
using (HttpWebResponse response = (HttpWebResponse)Future.Result)
{
try
{
using (Stream stream = response.GetResponseStream())
{
byte[] bytes;
if ("br".Equals(response.Headers[HttpResponseHeader.ContentEncoding], StringComparison.OrdinalIgnoreCase))
{
logger_.Verbose("decompressing brotli encoded resource.");
using (BrotliStream bs = new BrotliStream(stream, CompressionMode.Decompress))
{
bytes = CommonUtils.ReadToEnd(bs, MaxResourceSize);
}
}
else
{
bytes = CommonUtils.ReadToEnd(stream, MaxResourceSize);
}
rgResource_ = new RGridResource(Url, response.ContentType, bytes, logger_, "ResourceFuture");
logger_.Debug("{0} - size: {1} bytes", rgResource_, bytes.Length);
}
}
catch (Exception e)
{
logger_.Log("ERROR handling Future (URL: {0}) (Ref#: {1}): {2}", Url, Future.GetHashCode(), e);
}
finally
{
response.Close();
}
}
}
else if (Future.Exception.InnerException is WebException webException)
{
logger_.Log("Error downloading URL {0}: {1}", Url, webException);
rgResource_ = new RGridResource(Url, "unknown/error", new byte[0], logger_, webException.Message);
}
else
{
logger_.Log("Error while downloading URL {0}: {1}", Url, Future.Exception);
rgResource_ = new RGridResource(Url, "unknown/error", new byte[0], logger_, Future.Exception.Message);
}
}
}
return(rgResource_);
}
public RGridResource AsResource()
{
Logger.Verbose("enter");
string stringObjectMap = GetStringObjectMap_();
Logger.Verbose("creating RGridResource");
RGridResource gridResource = new RGridResource(Url, ContentType, Encoding.UTF8.GetBytes(stringObjectMap), Logger, "RGridDom - " + Msg);
Logger.Verbose("exit");
return(gridResource);
}
internal static TextualDataResource TryGetTextualData_(RGridResource blob, Uri baseUrl, Logger logger)
{
byte[] contentBytes = blob.Content;
string contentTypeStr = blob.ContentType;
logger.Verbose("enter - content length: {0} ; content type: {1}", contentBytes?.Length.ToString() ?? "<null>", contentTypeStr);
if (contentTypeStr == null)
{
return(null);
}
if (contentBytes.Length == 0)
{
return(null);
}
string[] parts = contentTypeStr.Split(';');
TextualDataResource tdr = new TextualDataResource();
if (parts.Length > 0)
{
tdr.MimeType = parts[0].ToLowerInvariant();
}
string charset = "UTF-8";
if (parts.Length > 1)
{
string[] keyVal = parts[1].Split('=');
string key = keyVal[0].Trim();
string val = keyVal[1].Trim().Trim('"');
if (key.Equals("charset", StringComparison.OrdinalIgnoreCase))
{
charset = val.ToUpper();
}
}
if (charset != null)
{
tdr.Data = Encoding.GetEncoding(charset).GetString(contentBytes);
}
tdr.Uri = blob.Url;
if (!tdr.Uri.IsAbsoluteUri)
{
tdr.Uri = new Uri(baseUrl, tdr.Uri);
}
tdr.OriginalData = blob.Content;
logger.Verbose("exit");
return(tdr);
}
internal RenderRequest[] PrepareDataForRG_(FrameData result)
{
logger_.Verbose("enter");
ConcurrentDictionary <string, RGridResource> allBlobs = new ConcurrentDictionary <string, RGridResource>();
HashSet <string> resourceUrls = new HashSet <string>();
ParseScriptResult_(result, allBlobs, resourceUrls);
logger_.Verbose("fetching {0} resources...", resourceUrls.Count);
//Fetch all resources
FetchAllResources_(allBlobs, resourceUrls);
logger_.Verbose("done fetching resources.");
List <RGridResource> written = AddBlobsToCache_(allBlobs);
logger_.Verbose("written {0} blobs to cache.", written.Count);
//Create RenderingRequest
//Parse allBlobs to mapping
ConcurrentDictionary <string, RGridResource> resourceMapping = new ConcurrentDictionary <string, RGridResource>();
foreach (KeyValuePair <string, RGridResource> kvp in allBlobs)
{
string urlStr = SanitizeUrl_(kvp.Key);
ResourceFuture resourceFuture = fetchedCacheMap_[urlStr];
Task <RGridResource> resourceTask = resourceFuture.Get(TimeSpan.FromSeconds(60));
RGridResource resource = resourceTask.Result;
resourceMapping.TryAdd(urlStr, resource); // TODO - ITAI
}
BuildAllRGDoms_(resourceMapping, result);
logger_.Verbose("resources count: {0}", resourceMapping.Count);
logger_.Verbose("cached resources count: {0}", cachedResourceMapping_.Count);
AppendAllCachedResources_(resourceMapping);
// Sort mapped resources by their URL for constant, testable results.
SortedDictionary <string, RGridResource> sortedResourceMapping = new SortedDictionary <string, RGridResource>(resourceMapping);
List <RenderRequest> allRequestsForRG = BuildRenderRequests_(result, sortedResourceMapping);
RenderRequest[] asArray = allRequestsForRG.ToArray();
logger_.Verbose("exit - returning renderRequest array of length: {0}", asArray.Length);
return(asArray);
}
private RGridResource ParseBlobToGridResource_(Uri baseUrl, BlobData blob)
{
byte[] content = null;
if (blob.Value != null)
{
content = Convert.FromBase64String(blob.Value);
}
Uri url = blob.Url;
if (!url.IsAbsoluteUri)
{
url = new Uri(baseUrl, url);
}
int? errorStatusCode = blob.ErrorStatusCode;
RGridResource resource = new RGridResource(url, blob.Type, content, logger_, "parseBlobToGridResource", errorStatusCode);
return(resource);
}
private void GetAndParseResource_(RGridResource blob, Uri baseUrl, HashSet <string> resourceUrls)
{
try
{
TextualDataResource tdr = TryGetTextualData_(blob, baseUrl, logger_);
switch (tdr?.MimeType)
{
case "text/css": ParseCSS_(tdr, resourceUrls, logger_); break;
case "image/svg+xml": ParseSVG_(tdr, resourceUrls, logger_); break;
}
}
catch (Exception ex)
{
logger_.Log("Error: " + ex);
logger_.Log("File name: " + blob.Url);
debugResourceWriter_.Write(blob);
}
}
private void AppendAllCachedResources_(ConcurrentDictionary <string, RGridResource> resourceMapping)
{
foreach (KeyValuePair <string, IEnumerable <string> > kvp in cachedResourceMapping_)
{
foreach (string url in kvp.Value)
{
if (resourceMapping.ContainsKey(url))
{
continue;
}
if (fetchedCacheMap_.TryGetValue(url, out ResourceFuture value))
{
RGridResource res = value.GetResource();
if (res != null)
{
resourceMapping.TryAdd(url, res);
}
}
}
}
}
public void Write(RGridResource value)
{
if (value.Content == null || value.Content.Length == 0 || value.ContentType == null)
{
return;
}
string filename;
lock (lockObject_)
{
if (!Directory.Exists(TargetFolder))
{
Directory.CreateDirectory(TargetFolder);
}
//filename = Path.Combine(TargetFolder, Interlocked.Increment(ref counter_) + "_" + value.Sha256);
string ext = value.ContentType;
string ext2 = null;
int semicolon = ext.IndexOf(";");
if (semicolon > -1)
{
ext = ext.Substring(0, semicolon);
}
if (extensions.TryGetValue(ext, out string[] mimeExtensions))
public void Write(RGridResource value)
{
// Do nothing
}
public ResourceFuture(RGridResource rgResource, Logger logger)
{
Url = rgResource.Url;
rgResource_ = rgResource;
logger_ = logger;
}
public PutFuture(Task <WebResponse> putFuture, RGridResource resource, RunningRender runningRender, IEyesConnector eyesConnector, Logger logger)
: this(resource, runningRender, eyesConnector, logger)
{
putFuture_ = putFuture;
}
private IDictionary <string, RGridResource> BuildAllRGDoms_(IDictionary <string, RGridResource> resourceMapping, FrameData currentFrame)
{
Uri baseUrl = currentFrame.Url;
logger_.Verbose("enter - baseUrl: " + baseUrl);
ConcurrentDictionary <string, RGridResource> mapping = new ConcurrentDictionary <string, RGridResource>();
ConcurrentDictionary <string, RGridResource> frames = new ConcurrentDictionary <string, RGridResource>();
logger_.Verbose("iterating {0} sub-frames", currentFrame.Frames.Count);
foreach (FrameData frame in currentFrame.Frames)
{
logger_.Verbose("iterating {0} sub-frame blobs", frame.Blobs.Count);
foreach (BlobData blob in frame.Blobs)
{
string urlStr = blob.Url.OriginalString;
if (resourceMapping.TryGetValue(urlStr, out RGridResource rGridResource))
{
mapping.TryAdd(urlStr, rGridResource);
}
}
logger_.Verbose("iterating {0} sub-frame resource urls", frame.ResourceUrls.Count);
foreach (Uri resourceUrl in frame.ResourceUrls)
{
string urlStr = resourceUrl.OriginalString;
if (resourceMapping.TryGetValue(urlStr, out RGridResource rGridResource))
{
mapping.TryAdd(urlStr, rGridResource);
}
}
Uri frameUrl = frame.Url;
/*if ("about:blank".Equals(frameUrl.ToString(), StringComparison.OrdinalIgnoreCase))
* {
* continue;
* }
* else */
if (!frameUrl.IsAbsoluteUri)
{
frameUrl = new Uri(baseUrl, frameUrl);
}
try
{
logger_.Verbose("entering recursion. url: {0}", frameUrl);
IDictionary <string, RGridResource> innerFrameResourceMap = BuildAllRGDoms_(resourceMapping, frame);
logger_.Verbose("exiting recursion. url: {0}", frameUrl);
foreach (KeyValuePair <string, RGridResource> kvp in innerFrameResourceMap)
{
mapping.TryAdd(kvp.Key, kvp.Value);
}
logger_.Verbose("mapped resources: {0}", mapping.Count);
// Sort mapped resources by their URL for constant, testable results.
SortedDictionary <string, RGridResource> sortedMapping = new SortedDictionary <string, RGridResource>(mapping);
RGridDom rGridDom = new RGridDom(frame.Cdt, sortedMapping, frameUrl, logger_, "buildAllRGDoms");
logger_.Verbose("adding resources to dictionary. url: {0}", frameUrl);
RGridResource frameResource = rGridDom.AsResource();
resourceMapping[frameUrl.OriginalString] = frameResource;
frames[frameUrl.OriginalString] = frameResource;
}
catch (Exception e)
{
logger_.Log("Error (10): " + e);
}
}
logger_.Verbose("exit");
return(frames);
}
