public void Setup() { IFormatter formatter = new BinaryFormatter(); _scrapeOne = (Scrape)formatter.Deserialize(GetResourceStream("QualityBot.Test.Tests.TestData.FakeAncestryDevScrape.bin")); _scrapeTwo = (Scrape)formatter.Deserialize(GetResourceStream("QualityBot.Test.Tests.TestData.FakeAncestryStageScrape.bin")); _comparer = new Comparer(); }
/// <summary> /// Compares the specified pages and returns an object detailing the differences. /// </summary> /// <param name="pageA">The first page.</param> /// <param name="pageB">The second page.</param> /// <returns>An object of type CompareResult.</returns> public Comparison Compare(Scrape pageA, Scrape pageB) { return ComputeDiff(pageA, pageB); }
// TODO: high value testing area private bool HasChanges(ScrapedElement element, Image pageScreenshotA, Image pageScreenshotB, Scrape pageA, Scrape pageB, out ElementChangeResult changes) { var correspondingScrapedElement = element.CorrespondingScrapedElement; changes = new ElementChangeResult(); var changed = false; // Location changed decimal percentageChange = 0; if (!correspondingScrapedElement.Location.Equals(element.Location)) { changed = true; changes.LocationChanges = GetLocationChanges(correspondingScrapedElement.Location, element.Location, out percentageChange); } changes.LocationPercentageChange = percentageChange; // Css changed percentageChange = 0; if (!correspondingScrapedElement.Css.DictionaryEqual(element.Css)) { changed = true; changes.CssChanges = GetCssChanges(correspondingScrapedElement.Css, element.Css, out percentageChange); } changes.CssPercentageChange = percentageChange; // Html changed percentageChange = 0; if (correspondingScrapedElement.Html != element.Html) { changed = true; changes.HtmlChanges = GetStringChanges(correspondingScrapedElement.Html, element.Html, out percentageChange); } changes.HtmlPercentageChange = percentageChange; // Text changed percentageChange = 0; if (correspondingScrapedElement.Text != element.Text) { changed = true; changes.TextChanges = GetStringChanges(correspondingScrapedElement.Text, element.Text, out percentageChange); } changes.TextPercentageChange = percentageChange; // Pixels changed changes.PixelChanges = GetPixelChanges(pageScreenshotA, pageScreenshotB, element, correspondingScrapedElement, pageA, pageB, out percentageChange); changes.PixelPercentageChange = percentageChange; if (percentageChange > 0M) { changed = true; } // Store location on screenshot changes.LocationOnScreenshot = element.LocationOnScreenshot; return changed; }
/// <summary> /// Calculates the pixel difference between two elements. /// </summary> /// <param name="psA">The screenshot of the first page.</param> /// <param name="psB">The screenshot of the second page.</param> /// <param name="eA">The first element.</param> /// <param name="eB">The second element.</param> /// <param name="pA">The information about the first page.</param> /// <param name="pB">The information about the second page.</param> /// <param name="percentageChange">The change as a percentage.</param> /// <returns>An object containing information about the pixel differences.</returns> private PixelChange GetPixelChanges(Image psA, Image psB, ScrapedElement eA, ScrapedElement eB, Scrape pA, Scrape pB, out decimal percentageChange) { PixelChange pixelChange = null; Image originalA = ImageUtil.CropImage(psA, eA.LocationOnScreenshot); Image originalB = ImageUtil.CropImage(psB, eB.LocationOnScreenshot); Region regionA = ImageUtil.GetClippedRegion(eA.LocationOnScreenshot, pA.Elements.Select(e => e.LocationOnScreenshot)); Region regionB = ImageUtil.GetClippedRegion(eB.LocationOnScreenshot, pB.Elements.Select(e => e.LocationOnScreenshot)); Bitmap clippedA = ImageUtil.GetClippedImage(new Size(eA.LocationOnScreenshot.Width, eA.LocationOnScreenshot.Height), originalA, regionA); Bitmap clippedB = ImageUtil.GetClippedImage(new Size(eB.LocationOnScreenshot.Width, eB.LocationOnScreenshot.Height), originalB, regionB); Bitmap diffMask = ImageUtil.BitmapDiff(clippedA, clippedB, _ia, out percentageChange); Bitmap fromRegionMask = ImageUtil.DrawRegionAsMasks(new Size(eA.LocationOnScreenshot.Width, eA.LocationOnScreenshot.Height), regionA, originalA, _ia); Bitmap toRegionMask = ImageUtil.DrawRegionAsMasks(new Size(eB.LocationOnScreenshot.Width, eB.LocationOnScreenshot.Height), regionB, originalB, _ia); if (percentageChange > 0 || eB.LocationOnScreenshot.Width != eA.LocationOnScreenshot.Width || eB.LocationOnScreenshot.Height != eA.LocationOnScreenshot.Height) { pixelChange = new PixelChange { From = originalA, FromClipped = clippedA, FromMask = fromRegionMask, To = originalB, ToClipped = clippedB, ToMask = toRegionMask, Diff = diffMask }; } regionA.Dispose(); regionB.Dispose(); return pixelChange; }
/// <summary> /// Returns information about the given element. /// </summary> /// <param name="pageScreenshot">The screenshot of the page containing the element.</param> /// <param name="scrapedElement">The element.</param> /// <param name="page">The page containing the element.</param> /// <returns>An ElementAddRemoveResult object.</returns> private ElementAddRemoveResult GetElementData(Image pageScreenshot, ScrapedElement scrapedElement, Scrape page) { var originalA = ImageUtil.CropImage(pageScreenshot, scrapedElement.LocationOnScreenshot); var regionA = ImageUtil.GetClippedRegion(scrapedElement.LocationOnScreenshot, page.Elements.Select(e => e.LocationOnScreenshot)); var clippedA = ImageUtil.GetClippedImage(new Size(scrapedElement.LocationOnScreenshot.Width, scrapedElement.LocationOnScreenshot.Height), originalA, regionA); var imageMask = ImageUtil.DrawRegionAsMasks(new Size(scrapedElement.LocationOnScreenshot.Width, scrapedElement.LocationOnScreenshot.Height), regionA, originalA, _ia); var add = new ElementAddRemoveResult { Attributes = scrapedElement.Attributes, Html = scrapedElement.Html, Text = scrapedElement.Text, Location = scrapedElement.LocationOnScreenshot, Tag = scrapedElement.Tag, Image = originalA, ImageClipped = clippedA, ImageMask = imageMask }; return add; }
private IEnumerable<ElementChangeResult> GetChanges(IEnumerable<ScrapedElement> matches, Image pageScreenshotA, Image pageScreenshotB, Scrape pageA, Scrape pageB) { foreach (var match in matches) { ElementChangeResult change; if (HasChanges(match, pageScreenshotA, pageScreenshotB, pageA, pageB, out change)) { match.HasChanges = true; yield return change; } } }
/// <summary> /// Attempts to determine which element on page A corresponds to which element on page B. /// </summary> /// <returns>A collection of mappings.</returns> private void CorrespondingElements(Scrape pageA, Scrape pageB, decimal maxDistance) { var elementMatches = new List<ElementMatch<ScrapedElement>>(); foreach (var element in pageA.Elements) { ScrapedElement match; Tuple<ElementMatch<ScrapedElement>, decimal>[] matches; if (_elementMapper.HasExactMatch(element, pageB.Elements, out match) || _elementMapper.HasIdMatch(element, pageB.Elements, out match)) { element.CorrespondingScrapedElement = match; match.CorrespondingScrapedElement = element; } else if (_elementMapper.HasSimilarElements(element, pageB.Elements, maxDistance, out matches)) { var elementMatch = new ElementMatch<ScrapedElement> { This = element, Matches = new Queue<Tuple<ElementMatch<ScrapedElement>, decimal>>(matches) }; elementMatch.SetToNext(); elementMatches.Add(elementMatch); } } // Resolve conflicts _conflictResolver.ResolveAllConflicts(elementMatches.ToArray()); foreach (var element in elementMatches.Where(m => m.Match != null)) { element.This.CorrespondingScrapedElement = element.Match.This; element.Match.This.CorrespondingScrapedElement = element.This; } }
/// <summary> /// Compares the specified pages and returns an object detailing the differences. /// </summary> /// <param name="pageA">The first page.</param> /// <param name="pageB">The second page.</param> /// <returns>An object of type Comparison.</returns> private Comparison ComputeDiff(Scrape pageA, Scrape pageB) { Comparison comparison; using (Image pageScreenshotA = ImageUtil.Base64ToImage(pageA.Screenshot), pageScreenshotB = ImageUtil.Base64ToImage(pageB.Screenshot)) { // Html diff var htmlDiff = GetHtmlDiffReport(pageA.Html, pageB.Html); // Pixel diff var pixelDiff = PixelDiff(pageScreenshotA, pageScreenshotB); // Match up elements var largestDiagonal = (decimal)Math.Max( Hypotenuse(pageScreenshotA.Width, pageScreenshotA.Height), Hypotenuse(pageScreenshotB.Width, pageScreenshotB.Height)); CorrespondingElements(pageA, pageB, largestDiagonal); var matches = pageA.Elements.Where(e => e.CorrespondingScrapedElement != null).ToArray(); // Get changes var changedItems = GetChanges(matches, pageScreenshotA, pageScreenshotB, pageA, pageB).ToArray(); var unchanged = matches.Count(m => !m.HasChanges); var changed = matches.Where(m => m.HasChanges).ToArray(); var deleted = pageA.Elements.Where(e => e.CorrespondingScrapedElement == null).ToArray(); var added = pageB.Elements.Where(e => e.CorrespondingScrapedElement == null).ToArray(); var removedItems = deleted.Select(e => GetElementData(pageScreenshotA, e, pageA)).ToArray(); var addedItems = added.Select(e => GetElementData(pageScreenshotB, e, pageB)).ToArray(); // Compute overall percentages var total = (added.Length + deleted.Length + unchanged + changed.Length) * 100; var addedDeleted = (added.Length + deleted.Length) * 100; var cssChangePercentage = TotalChangePercentage(total, changedItems.Sum(i => i.CssPercentageChange), addedDeleted); var textChangePercentage = TotalChangePercentage(total, changedItems.Sum(i => i.TextPercentageChange), addedDeleted); var overallElementPositionChangePercentage = TotalChangePercentage(total, changedItems.Sum(i => i.LocationPercentageChange), addedDeleted); var htmlPercentChanged = TotalChangePercentage(total, changedItems.Sum(i => i.HtmlPercentageChange), addedDeleted); // Create Added/Changed elements and Deleted elements images var outlinedImages = OutlinedImagesAsBase64(pageScreenshotA, pageScreenshotB, added, deleted, changedItems).ToArray(); var scrapeHybridA = AssembleScrapeHybrid(pageA, true); var scrapeHybridB = AssembleScrapeHybrid(pageB, false); comparison = new Comparison { Scrapes = new[] { scrapeHybridA, scrapeHybridB }, Result = new PageResult { HtmlDiff = htmlDiff, Pixels = pixelDiff, ChangedItems = changedItems, AddedItems = addedItems, RemovedItems = removedItems, UnchangedItems = unchanged, CssChangePercentage = cssChangePercentage, TextChangePercentage = textChangePercentage, OverallElementPositionChangePercentage = overallElementPositionChangePercentage, Html = new HtmlResult { PercentChanged = htmlPercentChanged, Images = outlinedImages }, } }; } return comparison; }
private ScrapeHybrid AssembleScrapeHybrid(Scrape page, bool isBaseline) { return new ScrapeHybrid { IncludeJquerySelector = page.IncludeJquerySelector, ExcludeJquerySelector = page.ExcludeJquerySelector, Script = page.Script, BoundingRectangle = page.BoundingRectangle, Description = isBaseline ? "Baseline" : "Delta", IdString = page.IdString, Path = page.Path, Resources = page.Resources, Cookies = page.Cookies, Html = page.HtmlRef, Url = page.Url, Screenshot = page.ScreenshotRef, ViewportSize = page.ViewportSize, Browser = page.Browser, BrowserVersion = page.BrowserVersion, TimeStamp = page.TimeStamp, Platform = page.Platform }; }
public Scrape FakeScrape(FakeScrapeParams fakeScrapeParams) { fakeScrapeParams.Cookies = new List<string>(); #region Header/Cookie Content if (!fakeScrapeParams.Cookies.Any()) { fakeScrapeParams.Cookies.AddRange(FakeCookies()); } var headerOne = new List<string>() { "Content-Length:194", "Cache-Control:public, must-revalidate", "Content-Type:application/x-javascript", "Date:Thu, 20 Sep 2012 17:15:03 GMT", "ETag:JsJt380DknGc4kAEEn76og==" }; var headerTwo = new List<string>() { "Content-Length:17423", "Cache-Control:public, must-revalidate", "Content-Type:application/x-javascript", "Date:Thu, 20 Sep 2012 17:15:03 GMT", "ETag:qloGz7WY45YMKQ1Fmuuw8A==" }; var headerThree = new List<string>() { "Content-Length:2552", "Cache-Control:public, must-revalidate", "Content-Type:image/gif", "Date:Thu, 20 Sep 2012 17:15:03 GMT", "ETag:UAFdRlkmdsJ1EGIoGalWng==" }; #endregion if (fakeScrapeParams.Resources == null) { //Uri, statusCode, StatusDesc, Headers var first = GetSession().List<Resource>(3).First(1) .Impose(x => x.Uri, "http://c.mfcreativedev.com/webparts/banner/Banner.js?v=c5589edb") .Impose(x => x.StatusCode, HttpStatusCode.OK) .Impose(x => x.StatusDescription, "OK") .Impose(x => x.Headers, headerOne) .Next(1) .Impose(x => x.Uri, "http://c.mfcreativedev.com/webparts/header/HeaderV1_2.js?v=730f5c7b1") .Impose(x => x.StatusCode, HttpStatusCode.OK) .Impose(x => x.StatusDescription, "OK") .Impose(x => x.Headers, headerTwo) .Next(1) .Impose(x => x.Uri, "http://c.mfcreativedev.com/s/0/p/0/i/ances_logo.gif") .Impose(x => x.StatusCode, HttpStatusCode.OK) .Impose(x => x.StatusDescription, "OK") .Impose(x => x.Headers, headerThree) .All().Get().ToArray(); fakeScrapeParams.Resources = first; } if (fakeScrapeParams.Elements == null) { var elements = new List<ScrapedElement>(); var ele1 = new ScrapedElement() { Attributes = new Dictionary<string, string>() {{"id", "mngb"}}, CorrespondingScrapedElement = null, Css = new Dictionary<string, string>(), Html = "<div id=\"mngb\"></div>", Location = new Rectangle(0, 0, 800, 30), LocationOnScreenshot = new Rectangle(0, 0, 800, 30), Tag = "div", Text = "" }; elements.Add(ele1); fakeScrapeParams.Elements = new List<ScrapedElement>(elements); } var scr = new Scrape { Id = new ObjectId(fakeScrapeParams.Id), ExcludeJquerySelector = fakeScrapeParams.Exclude, IncludeJquerySelector = fakeScrapeParams.Include, Script = fakeScrapeParams.Script, BoundingRectangle = fakeScrapeParams.Bounding, Path = new StringAsReference {Value = fakeScrapeParams.Path}, Elements = fakeScrapeParams.Elements, Resources = fakeScrapeParams.Resources, Html = fakeScrapeParams.Html, HtmlRef = new StringAsReference {Value = fakeScrapeParams.HtmlRef}, Url = fakeScrapeParams.Url, Screenshot = fakeScrapeParams.ScreenShot, ScreenshotRef = new StringAsReference {Value = fakeScrapeParams.ScreenShotRef}, ViewportSize = fakeScrapeParams.ViewportSize == null ? new Size(800 , 600) : fakeScrapeParams.ViewportSize.Value, Browser = fakeScrapeParams.Browser, BrowserVersion = fakeScrapeParams.BrowserVersion, TimeStamp = fakeScrapeParams.TimeStamp == null ? DateTime.Now : fakeScrapeParams.TimeStamp.Value, Platform = fakeScrapeParams.Platform, Cookies = fakeScrapeParams.Cookies }; return scr; }
public Comparison CompareFakeScrapes(Scrape fakeScr1, Scrape fakeScr2) { var compare = new Comparer(); Comparison comparison = compare.Compare(fakeScr1, fakeScr2); return comparison; }
private void SerializeData(Scrape data) { IFormatter formatter = new BinaryFormatter(); Stream stream = new FileStream(@"C:\testme\FakeAncestryStageScrape.bin", FileMode.Create, FileAccess.Write, FileShare.None); formatter.Serialize(stream, data); stream.Close(); }
private Scrape AssembleScrape(Request request, PageData pageData, Resource[] pageResources) { var screenshotBase64 = ImageUtil.ImageToBase64(pageData.Screenshot, ImageFormat.Png); var scrape = new Scrape { ExcludeJquerySelector = request.ExcludeJquerySelector, IncludeJquerySelector = request.IncludeJquerySelector, BoundingRectangle = request.BoundingRectangle, Script = request.Script, Url = pageData.Url, TimeStamp = DateTime.Now, Browser = pageData.BrowserName.ToLower(), BrowserVersion = pageData.BrowserVersion.ToLower(), ViewportSize = pageData.Size, Platform = pageData.Platform.ToLower(), Resources = pageResources, Html = CleanHtml(pageData.Html), Screenshot = screenshotBase64, Cookies = pageData.Cookies.ToList() }; return scrape; }
private void ConstrainElementsToBoundingRectangle(Scrape page, Rectangle clipRect) { foreach (var e in page.Elements) { var r = e.LocationOnScreenshot; r.X -= clipRect.X; r.Y -= clipRect.Y; e.LocationOnScreenshot = r; } }
public void Setup() { _path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, _path); Directory.CreateDirectory(_path); _scrape = new QBFake().FakeScrape(new FakeScrapeParams()); }