private void DownloadResource_(string url, List <Task <RGridResource> > tasks) { url = SanitizeUrl_(url); tasks.Add(Task.Run(async() => { if (!fetchedCacheMap_.TryGetValue(url, out ResourceFuture future)) { future = DownloadResourceFuture_(url); if (fetchedCacheMap_.TryAdd(url, future)) { logger_.Verbose("fetchedCacheMap_.Add({0})", url); } } RGridResource r = await future.Get(TimeSpan.FromSeconds(60)); debugResourceWriter_.Write(r); HashSet <string> newResources = new HashSet <string>(); GetAndParseResource_(r, r.Url, newResources); string sanitizedUrl = SanitizeUrl_(r.Url.OriginalString); cachedResourceMapping_.TryAdd(sanitizedUrl, newResources); logger_.Verbose("adding {0} more resources to download queue.", newResources.Count); foreach (string rUrl in newResources) { sanitizedUrl = SanitizeUrl_(rUrl); if (fetchedCacheMap_.ContainsKey(sanitizedUrl)) { continue; } resourcesToFetch_.Enqueue(sanitizedUrl); logger_.Verbose("added url to download: {0}", sanitizedUrl); } return(r); })); }
public PutFuture(RGridResource resource, RunningRender runningRender, IEyesConnector eyesConnector, Logger logger) { resource_ = resource; runningRender_ = runningRender; eyesConnector_ = eyesConnector; logger_ = logger; }
internal void FetchAllResources_(IDictionary <string, RGridResource> allBlobs, HashSet <string> resourceUrls) { logger_.Verbose("enter"); resourcesToFetch_ = new Queue <string>(resourceUrls); while (resourcesToFetch_.Count > 0) { var tasks = new List <Task <RGridResource> >(); while (resourcesToFetch_.Count > 0) { string url = resourcesToFetch_.Dequeue(); DownloadResource_(url, tasks); } while (tasks.Any(t => !t.IsCompleted)) { Task.WaitAll(tasks.ToArray()); } foreach (Task <RGridResource> task in tasks) { RGridResource resource = task.Result; string url = resource.Url.OriginalString; if (!allBlobs.ContainsKey(url)) { allBlobs.Add(url, resource); } } } logger_.Verbose("exit"); }
public async Task <RGridResource> Get(TimeSpan timeout) { if (rgResource_ == null) { logger_.Debug("{0} Future ref#: {1}", Url, Future.GetHashCode()); if (await Task.WhenAny(Future, Task.Delay(timeout)) == Future) { if (Future.Exception == null) { using (HttpWebResponse response = (HttpWebResponse)Future.Result) { try { using (Stream stream = response.GetResponseStream()) { byte[] bytes; if ("br".Equals(response.Headers[HttpResponseHeader.ContentEncoding], StringComparison.OrdinalIgnoreCase)) { logger_.Verbose("decompressing brotli encoded resource."); using (BrotliStream bs = new BrotliStream(stream, CompressionMode.Decompress)) { bytes = CommonUtils.ReadToEnd(bs, MaxResourceSize); } } else { bytes = CommonUtils.ReadToEnd(stream, MaxResourceSize); } rgResource_ = new RGridResource(Url, response.ContentType, bytes, logger_, "ResourceFuture"); logger_.Debug("{0} - size: {1} bytes", rgResource_, bytes.Length); } } catch (Exception e) { logger_.Log("ERROR handling Future (URL: {0}) (Ref#: {1}): {2}", Url, Future.GetHashCode(), e); } finally { response.Close(); } } } else if (Future.Exception.InnerException is WebException webException) { logger_.Log("Error downloading URL {0}: {1}", Url, webException); rgResource_ = new RGridResource(Url, "unknown/error", new byte[0], logger_, webException.Message); } else { logger_.Log("Error while downloading URL {0}: {1}", Url, Future.Exception); rgResource_ = new RGridResource(Url, "unknown/error", new byte[0], logger_, Future.Exception.Message); } } } return(rgResource_); }
public RGridResource AsResource() { Logger.Verbose("enter"); string stringObjectMap = GetStringObjectMap_(); Logger.Verbose("creating RGridResource"); RGridResource gridResource = new RGridResource(Url, ContentType, Encoding.UTF8.GetBytes(stringObjectMap), Logger, "RGridDom - " + Msg); Logger.Verbose("exit"); return(gridResource); }
internal static TextualDataResource TryGetTextualData_(RGridResource blob, Uri baseUrl, Logger logger) { byte[] contentBytes = blob.Content; string contentTypeStr = blob.ContentType; logger.Verbose("enter - content length: {0} ; content type: {1}", contentBytes?.Length.ToString() ?? "<null>", contentTypeStr); if (contentTypeStr == null) { return(null); } if (contentBytes.Length == 0) { return(null); } string[] parts = contentTypeStr.Split(';'); TextualDataResource tdr = new TextualDataResource(); if (parts.Length > 0) { tdr.MimeType = parts[0].ToLowerInvariant(); } string charset = "UTF-8"; if (parts.Length > 1) { string[] keyVal = parts[1].Split('='); string key = keyVal[0].Trim(); string val = keyVal[1].Trim().Trim('"'); if (key.Equals("charset", StringComparison.OrdinalIgnoreCase)) { charset = val.ToUpper(); } } if (charset != null) { tdr.Data = Encoding.GetEncoding(charset).GetString(contentBytes); } tdr.Uri = blob.Url; if (!tdr.Uri.IsAbsoluteUri) { tdr.Uri = new Uri(baseUrl, tdr.Uri); } tdr.OriginalData = blob.Content; logger.Verbose("exit"); return(tdr); }
internal RenderRequest[] PrepareDataForRG_(FrameData result) { logger_.Verbose("enter"); ConcurrentDictionary <string, RGridResource> allBlobs = new ConcurrentDictionary <string, RGridResource>(); HashSet <string> resourceUrls = new HashSet <string>(); ParseScriptResult_(result, allBlobs, resourceUrls); logger_.Verbose("fetching {0} resources...", resourceUrls.Count); //Fetch all resources FetchAllResources_(allBlobs, resourceUrls); logger_.Verbose("done fetching resources."); List <RGridResource> written = AddBlobsToCache_(allBlobs); logger_.Verbose("written {0} blobs to cache.", written.Count); //Create RenderingRequest //Parse allBlobs to mapping ConcurrentDictionary <string, RGridResource> resourceMapping = new ConcurrentDictionary <string, RGridResource>(); foreach (KeyValuePair <string, RGridResource> kvp in allBlobs) { string urlStr = SanitizeUrl_(kvp.Key); ResourceFuture resourceFuture = fetchedCacheMap_[urlStr]; Task <RGridResource> resourceTask = resourceFuture.Get(TimeSpan.FromSeconds(60)); RGridResource resource = resourceTask.Result; resourceMapping.TryAdd(urlStr, resource); // TODO - ITAI } BuildAllRGDoms_(resourceMapping, result); logger_.Verbose("resources count: {0}", resourceMapping.Count); logger_.Verbose("cached resources count: {0}", cachedResourceMapping_.Count); AppendAllCachedResources_(resourceMapping); // Sort mapped resources by their URL for constant, testable results. SortedDictionary <string, RGridResource> sortedResourceMapping = new SortedDictionary <string, RGridResource>(resourceMapping); List <RenderRequest> allRequestsForRG = BuildRenderRequests_(result, sortedResourceMapping); RenderRequest[] asArray = allRequestsForRG.ToArray(); logger_.Verbose("exit - returning renderRequest array of length: {0}", asArray.Length); return(asArray); }
private RGridResource ParseBlobToGridResource_(Uri baseUrl, BlobData blob) { byte[] content = null; if (blob.Value != null) { content = Convert.FromBase64String(blob.Value); } Uri url = blob.Url; if (!url.IsAbsoluteUri) { url = new Uri(baseUrl, url); } int? errorStatusCode = blob.ErrorStatusCode; RGridResource resource = new RGridResource(url, blob.Type, content, logger_, "parseBlobToGridResource", errorStatusCode); return(resource); }
private void GetAndParseResource_(RGridResource blob, Uri baseUrl, HashSet <string> resourceUrls) { try { TextualDataResource tdr = TryGetTextualData_(blob, baseUrl, logger_); switch (tdr?.MimeType) { case "text/css": ParseCSS_(tdr, resourceUrls, logger_); break; case "image/svg+xml": ParseSVG_(tdr, resourceUrls, logger_); break; } } catch (Exception ex) { logger_.Log("Error: " + ex); logger_.Log("File name: " + blob.Url); debugResourceWriter_.Write(blob); } }
private void AppendAllCachedResources_(ConcurrentDictionary <string, RGridResource> resourceMapping) { foreach (KeyValuePair <string, IEnumerable <string> > kvp in cachedResourceMapping_) { foreach (string url in kvp.Value) { if (resourceMapping.ContainsKey(url)) { continue; } if (fetchedCacheMap_.TryGetValue(url, out ResourceFuture value)) { RGridResource res = value.GetResource(); if (res != null) { resourceMapping.TryAdd(url, res); } } } } }
public void Write(RGridResource value) { if (value.Content == null || value.Content.Length == 0 || value.ContentType == null) { return; } string filename; lock (lockObject_) { if (!Directory.Exists(TargetFolder)) { Directory.CreateDirectory(TargetFolder); } //filename = Path.Combine(TargetFolder, Interlocked.Increment(ref counter_) + "_" + value.Sha256); string ext = value.ContentType; string ext2 = null; int semicolon = ext.IndexOf(";"); if (semicolon > -1) { ext = ext.Substring(0, semicolon); } if (extensions.TryGetValue(ext, out string[] mimeExtensions))
public void Write(RGridResource value) { // Do nothing }
public ResourceFuture(RGridResource rgResource, Logger logger) { Url = rgResource.Url; rgResource_ = rgResource; logger_ = logger; }
public PutFuture(Task <WebResponse> putFuture, RGridResource resource, RunningRender runningRender, IEyesConnector eyesConnector, Logger logger) : this(resource, runningRender, eyesConnector, logger) { putFuture_ = putFuture; }
private IDictionary <string, RGridResource> BuildAllRGDoms_(IDictionary <string, RGridResource> resourceMapping, FrameData currentFrame) { Uri baseUrl = currentFrame.Url; logger_.Verbose("enter - baseUrl: " + baseUrl); ConcurrentDictionary <string, RGridResource> mapping = new ConcurrentDictionary <string, RGridResource>(); ConcurrentDictionary <string, RGridResource> frames = new ConcurrentDictionary <string, RGridResource>(); logger_.Verbose("iterating {0} sub-frames", currentFrame.Frames.Count); foreach (FrameData frame in currentFrame.Frames) { logger_.Verbose("iterating {0} sub-frame blobs", frame.Blobs.Count); foreach (BlobData blob in frame.Blobs) { string urlStr = blob.Url.OriginalString; if (resourceMapping.TryGetValue(urlStr, out RGridResource rGridResource)) { mapping.TryAdd(urlStr, rGridResource); } } logger_.Verbose("iterating {0} sub-frame resource urls", frame.ResourceUrls.Count); foreach (Uri resourceUrl in frame.ResourceUrls) { string urlStr = resourceUrl.OriginalString; if (resourceMapping.TryGetValue(urlStr, out RGridResource rGridResource)) { mapping.TryAdd(urlStr, rGridResource); } } Uri frameUrl = frame.Url; /*if ("about:blank".Equals(frameUrl.ToString(), StringComparison.OrdinalIgnoreCase)) * { * continue; * } * else */ if (!frameUrl.IsAbsoluteUri) { frameUrl = new Uri(baseUrl, frameUrl); } try { logger_.Verbose("entering recursion. url: {0}", frameUrl); IDictionary <string, RGridResource> innerFrameResourceMap = BuildAllRGDoms_(resourceMapping, frame); logger_.Verbose("exiting recursion. url: {0}", frameUrl); foreach (KeyValuePair <string, RGridResource> kvp in innerFrameResourceMap) { mapping.TryAdd(kvp.Key, kvp.Value); } logger_.Verbose("mapped resources: {0}", mapping.Count); // Sort mapped resources by their URL for constant, testable results. SortedDictionary <string, RGridResource> sortedMapping = new SortedDictionary <string, RGridResource>(mapping); RGridDom rGridDom = new RGridDom(frame.Cdt, sortedMapping, frameUrl, logger_, "buildAllRGDoms"); logger_.Verbose("adding resources to dictionary. url: {0}", frameUrl); RGridResource frameResource = rGridDom.AsResource(); resourceMapping[frameUrl.OriginalString] = frameResource; frames[frameUrl.OriginalString] = frameResource; } catch (Exception e) { logger_.Log("Error (10): " + e); } } logger_.Verbose("exit"); return(frames); }