public ActionResult IndexReversed()
{
// Get "Banner1"
var Banner1 = PageRetriever.Retrieve <Banner>(applyQueryParametersAction: query =>
query.Path("/Banners/Banner-1")
).FirstOrDefault();
// Get Items to look up on
int MobileCategoryID = CategoryInfoProvider.Get("Mobile", 0).CategoryID;
List <int> WesternUSOrEasternUSCategoryIDs = CategoryInfoProvider.Get()
.WhereIn(nameof(CategoryInfo.CategoryName), new string[] { "RegionWestUSA", "RegionEasternUSA" })
.Select(x => x.CategoryID).ToList();
int FooAID = FooInfoProvider.Get("FooA").FooID;
int BazAID = BazInfoProvider.Get("BazA").BazID;
int BarAID = BarInfoProvider.Get("BarA").BarID;
int BlahCategoryID = CategoryInfoProvider.Get("BlahCategory", 0).CategoryID;
TestReverseViewModel model = new TestReverseViewModel();
// Get Test pages related to banner 1
model.Banner1Pages = PageRetriever.Retrieve <Testing>(query =>
query.InRelationWithOrder(Banner1.NodeID, "Banners", ReverseRelationship: true)
).ToList();
model.MobilePages = PageRetriever.Retrieve <Testing>(query =>
query.TreeCategoryCondition(new object[] { MobileCategoryID })
)
.ToList();
model.WesternUSOrEasternUSPages = PageRetriever.Retrieve <Testing>(query =>
query.BindingCategoryCondition <Testing, NodeRegionInfo>(BindingConditionType.FilterParentsByChildren, WesternUSOrEasternUSCategoryIDs.Cast <object>())
)
.ToList();
model.FooAPages = PageRetriever.Retrieve <Testing>(query =>
query.BindingCondition <Testing, NodeFooInfo>(BindingConditionType.FilterParentsByChildren, new object[] { FooAID })
)
.ToList();
model.BazAPages = PageRetriever.Retrieve <Testing>(query =>
query.BindingCondition <Testing, NodeBazInfo>(BindingConditionType.FilterParentsByChildren, new object[] { BazAID })
)
.ToList();
model.BarAFoos = FooInfoProvider.Get()
.BindingCondition <FooInfo, FooBarInfo>(BindingConditionType.FilterParentsByChildren, new object[] { BarAID })
.ToList();
model.BazAFoos = FooInfoProvider.Get()
.BindingCondition <FooInfo, FooBazInfo>(BindingConditionType.FilterParentsByChildren, new object[] { BarAID })
.ToList();
model.BlahCategoryFoos = FooInfoProvider.Get()
.BindingCategoryCondition <FooInfo, FooCategoryInfo>(BindingConditionType.FilterParentsByChildren, new object[] { BlahCategoryID })
.ToList();
return(View("IndexReversed", model));
}
public void PageRetrievalWithAuthenticationWorks()
{
// arrange
const string username = "******";
const string password = "******";
var authValue = $"Basic {Convert.ToBase64String(Encoding.Default.GetBytes($"{username}:{password}"))}";
const string testPath = "/test";
const string url = "http://localhost:8654";
var fullUrl = $"{url}{testPath}";
var httpMock = HttpMockRepository.At(url);
httpMock.Stub(t => t.Get(testPath))
.Return("")
.OK();
var sut = new PageRetriever(new HttpClientFactory());
// act
var result = sut.RetrieveHtmlPageAsync(fullUrl, username, password).Result;
// assert
result.Should().NotBeNull();
var headers = httpMock.AssertWasCalled(t => t.Get(testPath))
.LastRequest()
.RequestHead
.Headers
.Should()
.Contain("Authorization", authValue);
}
public ActionResult IndexSimplified()
{
// Get current page
var CurrentTestPage = DataRetriever.Retrieve <Testing>().Page;
TestViewModel model = new TestViewModel();
model.Banners = PageRetriever.Retrieve <Banner>(query =>
query.InRelationWithOrder(CurrentTestPage.NodeID, "Banners")
).ToList();
model.Categories = CategoryInfoProvider.Get()
.BindingCondition <CategoryInfo, TreeCategoryInfo>(BindingConditionType.FilterChildrenByParents, new object[] { CurrentTestPage.NodeID })
.ToList();
model.Regions = CategoryInfoProvider.Get()
.InCustomRelationshipWithOrder <CategoryInfo, NodeRegionInfo>(BindingQueryType.GetChildrenByParent, CurrentTestPage.NodeID)
.ToList();
model.Foos = FooInfoProvider.Get()
.InCustomRelationshipWithOrder <FooInfo, NodeFooInfo>(BindingQueryType.GetChildrenByParentOrdered, CurrentTestPage.NodeID)
.ToList();
model.Bazs = BazInfoProvider.Get()
.InCustomRelationshipWithOrder <BazInfo, NodeBazInfo>(BindingQueryType.GetChildrenByParent, CurrentTestPage.NodeID)
.ToList();
// Get Foo Bars
foreach (var Foo in model.Foos)
{
model.FooBars.Add(Foo.FooID,
BarInfoProvider.Get()
.InCustomRelationshipWithOrder <BarInfo, FooBarInfo>(BindingQueryType.GetChildrenByParentOrdered, Foo.FooID)
.ToList()
);
}
// Get Foo Baz
foreach (var Foo in model.Foos)
{
model.FooBazs.Add(Foo.FooID,
BazInfoProvider.Get()
.InCustomRelationshipWithOrder <BazInfo, FooBazInfo>(BindingQueryType.GetChildrenByParent, Foo.FooID)
.ToList()
);
}
// Get Foo Categories
foreach (var Foo in model.Foos)
{
model.FooCategories.Add(Foo.FooID,
CategoryInfoProvider.Get()
.InCustomRelationshipWithOrder <CategoryInfo, FooCategoryInfo>(BindingQueryType.GetChildrenByParentOrdered, Foo.FooID)
.ToList()
);
}
return(View("Index", model));
}
private void ProcessProductListPage(ProcessQueueItem pqi)
{
if (cancel)
{
return;
}
var wi = (ExtWareInfo)pqi.Item;
var html = PageRetriever.ReadFromServer(pqi.URL);
var doc = CreateDoc(html);
pqi.Processed = true;
var products = doc.DocumentNode.SelectNodes("//ul[contains(@class,'ProductList')]/li");
if (products == null)
{
Log.Error(products, new NullReferenceException());
return;
}
foreach (var product in products)
{
var a = product.SelectSingleNode(".//div[@class='ProductDetails']/strong/a[@href]");
var name = a.InnerTextOrNull();
if (string.IsNullOrWhiteSpace(name))
{
Log.Warn(product, new NullReferenceException());
continue;
}
var url = a.AttributeOrNull("href");
wi.Url = url;
lock (this)
{
lstProcessQueue.Add(new ProcessQueueItem
{
Item = wi,
Name = name,
ItemType = ProcessLevels.ProductDetails,
URL = url
});
}
}
OnItemLoaded(null);
pqi.Processed = true;
MessagePrinter.PrintMessage("Product list processed");
StartOrPushPropertiesThread();
}
private void ProcessCategoryListPage(ProcessQueueItem pqi)
{
if (cancel)
{
return;
}
var html = PageRetriever.ReadFromServer(pqi.URL);
var doc = CreateDoc(html);
pqi.Processed = true;
var categories = doc.DocumentNode.SelectNodes("//ul[@class='category-list']/li/ul/li");
if (categories == null)
{
Log.Error(categories, new NullReferenceException());
return;
}
foreach (var category in categories)
{
var a = category.SelectSingleNode(".//a[@href]");
var name = a.InnerTextOrNull();
if (string.IsNullOrWhiteSpace(name))
{
Log.Warn(category, new NullReferenceException());
continue;
}
var wi = new ExtWareInfo
{
Category = name
};
var url = a.AttributeOrNull("href");
lock (this)
{
lstProcessQueue.Add(new ProcessQueueItem
{
Item = wi,
ItemType = ProcessLevels.SubCategory,
URL = Url + url,
Name = name
});
}
}
OnItemLoaded(null);
pqi.Processed = true;
MessagePrinter.PrintMessage("Category list processed");
StartOrPushPropertiesThread();
}
private TreeNode GetPage(PartialWidgetPageWidgetModel Properties, bool DocumentIDAndClassOnly = false)
{
string Culture = !string.IsNullOrWhiteSpace(Properties.Culture) ? Properties.Culture : System.Globalization.CultureInfo.CurrentCulture.Name;
if (Properties.PageSelectionMode.Equals(PartialWidgetPageWidgetModel._PageSelectionMode_Path) && !string.IsNullOrWhiteSpace(Properties.Path.FirstOrDefault()?.NodeAliasPath))
{
string Path = Properties.Path.FirstOrDefault().NodeAliasPath;
string SiteName = !string.IsNullOrWhiteSpace(Properties.SiteName) ? Properties.SiteName : SiteContext.CurrentSiteName;
// Convert path / page to url
return(PageRetriever.RetrieveMultiple(query =>
{
query.Path(Properties.Path.FirstOrDefault().NodeAliasPath, PathTypeEnum.Single)
.Culture(Culture)
.CombineWithDefaultCulture()
.CombineWithAnyCulture()
.OnSite(SiteName)
.Published(false);
if (DocumentIDAndClassOnly)
{
query.Columns(nameof(TreeNode.DocumentID), nameof(TreeNode.ClassName));
}
}
, cache =>
cache.Key($"PartialWidgetPageWidget_GetPage|{Path}|{Culture}|{SiteName}|{DocumentIDAndClassOnly}")
.Dependencies((pages, deps) => deps.Pages(pages))
).FirstOrDefault());
}
else if (Properties.PageSelectionMode.Equals(PartialWidgetPageWidgetModel._PageSelectionMode_ByNodeGuid) && Properties.Page.Count() > 0 && Properties.Page.First().NodeGuid != Guid.Empty)
{
// Convert path / page to url
return(PageRetriever.RetrieveMultiple(query =>
{
query.WhereEquals(nameof(TreeNode.NodeGUID), Properties.Page.First().NodeGuid)
.Culture(Culture)
.CombineWithDefaultCulture()
.CombineWithAnyCulture()
.Published(false);
if (DocumentIDAndClassOnly)
{
query.Columns(nameof(TreeNode.DocumentID), nameof(TreeNode.ClassName));
}
}
, cache =>
cache.Key($"PartialWidgetPageWidget_GetPage|{Properties.Page.First().NodeGuid}|{Culture}|{DocumentIDAndClassOnly}")
.Dependencies((pages, deps) => deps.Pages(pages))
).FirstOrDefault());
}
else
{
return(null);
}
}
public void Initialize(bool pagesAreInMemory, int loadCount)
{
stopWordsDK = GetStopWords(@"C:\Users\Jacob\Desktop\Index\stopwordsDK.txt");
this.pageDB = DI.pageDB;
this.ranker = DI.ranker;
//if we just crawled, the webpages are in memory, and we dont read from file
if (!pagesAreInMemory)
{
pageDB.LoadRawPagesFromFiles(loadCount);
}
this.tokenizer = DI.tokenizer;
this.termConstructor = DI.termContructor;
this.pageRetriever = DI.pageRetriever;
this.indexCreator = DI.indexCreator;
}
public static void Initialize()
{
ranker = new Ranker.Ranker();
indexer = new Indexer.Indexer();
urlFilter = new UrlFilter();
urlFrontier = new UrlFrontier();
indexCreator = new IndexCreator();
pageRetriever = new PageRetriever();
termContructor = new TermConstructor();
tokenizer = new Tokenizer();
webCrawler = new WebCrawler.WebCrawler();
pageDB = new PageDB();
DPC = new DuplicatePageChecker();
DUC = new DuplicateURLChecker();
pageFetcher = new PageFetcher();
pageParser = new PageParser();
}
private void ProcessDetailsPage(ProcessQueueItem pqi)
{
if (cancel)
{
return;
}
var wi = (ExtWareInfo)pqi.Item;
var html = PageRetriever.ReadFromServer(pqi.URL);
var doc = CreateDoc(html);
pqi.Processed = true;
var details = doc.DocumentNode.SelectSingleNode("//div[@class='BlockContent']");
if (details == null)
{
Log.Error(details, new NullReferenceException());
return;
}
var images = details.SelectNodes("//div[@class='ProductTinyImageList']/ul/li/div[@class='TinyOuterDiv']/div/a[@rel]")
.Select(s => JsonConvert.DeserializeObject <ImageObject>(s.AttributeOrNull("rel")));
var item = new ExtWareInfo
{
Category = wi.Category,
SubCategory = wi.SubCategory,
Page = wi.Page,
Url = wi.Url,
Title = details.SelectSingleNode("//h1").InnerTextOrNull(),
Description = details.SelectSingleNode("//div[@class='ProductDescriptionContainer']").InnerTextOrNull(),
Price = ParsePrice(details.SelectSingleNode("//em[@class='ProductPrice VariationProductPrice']").InnerTextOrNull()),
Weight = ParseDouble(details.SelectSingleNode("//span[@class='VariationProductWeight']").InnerTextOrNull()),
Images = String.Join(";", images.Select(s => s.largeimage))
};
AddWareInfo(item);
OnItemLoaded(null);
MessagePrinter.PrintMessage("Product '" + item.Title + "' details processed");
StartOrPushPropertiesThread();
}
public void BasicPageRetrievalWorks()
{
// arrange
const string testPath = "/test";
const string url = "http://localhost:8654";
var fullUrl = $"{url}{testPath}";
var httpMock = HttpMockRepository.At(url);
httpMock.Stub(t => t.Get(testPath))
.Return("")
.OK();
var sut = new PageRetriever(new HttpClientFactory());
// act
var result = sut.RetrieveHtmlPageAsync(fullUrl).Result;
// assert
result.Should().NotBeNull();
}
public void RetrievePageAsString()
{
// arrange
const string testPath = "/test";
const string url = "http://localhost:8654";
var fullUrl = $"{url}{testPath}";
var httpMock = HttpMockRepository.At(url);
httpMock.Stub(t => t.Get(testPath))
.Return("Hello World")
.OK();
var sut = new PageRetriever(new HttpClientFactory());
// act
var result = sut.RetrieveDocumentAsStringAsync(fullUrl, null, null).Result;
// assert
result.ResponseStatusCode.Should().Be(HttpStatusCode.OK);
result.Success.Should().BeTrue();
result.RetrievedBody.Should().Be("Hello World");
}
private int GetDocumentID(ShareableContentWidgetProperties Properties)
{
if (Properties.Pages == null || !Properties.Pages.Any(x => true))
{
return(0);
}
string culture = !string.IsNullOrWhiteSpace(Properties.Culture) ? Properties.Culture : System.Globalization.CultureInfo.CurrentCulture.Name;
return(ProgressiveCache.Load(cs =>
{
var FoundPage = PageRetriever.Retrieve <ShareableContent>(query =>
{
query.WhereEquals(nameof(TreeNode.NodeGUID), Properties.Pages.FirstOrDefault().NodeGuid)
.Culture(culture)
.CombineWithDefaultCulture()
.CombineWithAnyCulture()
.Columns(nameof(TreeNode.DocumentID));
}
).FirstOrDefault();
return (FoundPage != null ? FoundPage.DocumentID : 0);
}, new CacheSettings(1440, "GetShareableContentWidget", Properties.Pages.FirstOrDefault().NodeGuid.ToString(), culture)));
}
public override void Init(TagHelperContext context)
{
if (!string.IsNullOrWhiteSpace(RelativeUrl))
{
AjaxUrl = RelativeUrl;
}
else if (Page != null)
{
AjaxUrl = DocumentURLProvider.GetUrl(Page);
}
else if (Documentid > 0)
{
// Get Document
Page = PageRetriever.RetrieveMultiple(query =>
query.WhereEquals(nameof(TreeNode.DocumentID), Documentid)
.Published(false)
, cache =>
cache.Key($"PartialWidgetPageAjaxTagHelperGetDocument|{Documentid}")
.Dependencies((pages, deps) => deps.Pages(pages))
).FirstOrDefault();
if (Page != null)
{
AjaxUrl = DocumentURLProvider.GetUrl(Page);
}
}
if (!string.IsNullOrWhiteSpace(AjaxUrl))
{
Render = true;
// Append Special Url parameter
AjaxUrl += $"{(AjaxUrl.IndexOf('?') == -1 ? '?' : '&')}{PartialWidgetPageHelper.GetPartialUrlParameter()}=true";
}
else
{
Render = false;
}
base.Init(context);
}
public ActionResult Index()
{
// Get current page
var CurrentTestPage = DataRetriever.Retrieve <Testing>().Page;
TestViewModel model = new TestViewModel();
model.Banners = PageRetriever.Retrieve <Banner>(query =>
query.InRelationWithOrder(CurrentTestPage.NodeID, "Banners")
).ToList();
//PageRetriever.Retrieve<Banner>(query => query.InCustomRelationshipWithOrder(new BarNodeInfo()));
model.Categories = CategoryInfoProvider.Get()
.BindingCondition(TreeCategoryInfo.OBJECT_TYPE, "cms.node", nameof(CategoryInfo.CategoryID), nameof(TreeCategoryInfo.CategoryID), nameof(TreeCategoryInfo.NodeID), new string[] { CurrentTestPage.NodeID.ToString() })
.ToList();
model.Regions = CategoryInfoProvider.Get()
.InCustomRelationshipWithOrder(NodeRegionInfo.OBJECT_TYPE, "cms.node", CurrentTestPage.NodeID, nameof(CategoryInfo.CategoryID), nameof(NodeRegionInfo.NodeRegionNodeID), nameof(NodeRegionInfo.NodeRegionCategoryID))
.ToList();
model.Foos = FooInfoProvider.Get()
.InCustomRelationshipWithOrder(NodeFooInfo.OBJECT_TYPE, "cms.node", CurrentTestPage.NodeID, nameof(FooInfo.FooID), nameof(NodeFooInfo.NodeFooNodeID), nameof(NodeFooInfo.NodeFooFooID), nameof(NodeFooInfo.NodeFooOrder))
.ToList();
model.Bazs = BazInfoProvider.Get()
.InCustomRelationshipWithOrder(NodeBazInfo.OBJECT_TYPE, "cms.node", CurrentTestPage.NodeID, nameof(BazInfo.BazID), nameof(NodeBazInfo.NodeBazNodeID), nameof(NodeBazInfo.NodeBazBazID))
.ToList();
// Get Foo Bars
foreach (var Foo in model.Foos)
{
model.FooBars.Add(Foo.FooID,
BarInfoProvider.Get()
.InCustomRelationshipWithOrder(FooBarInfo.OBJECT_TYPE, BarInfo.OBJECT_TYPE, Foo.FooID, nameof(BarInfo.BarID), nameof(FooBarInfo.FooBarFooID), nameof(FooBarInfo.FooBarBarID), nameof(FooBarInfo.FooBarOrder))
.ToList()
);
}
// Get Foo Baz
foreach (var Foo in model.Foos)
{
model.FooBazs.Add(Foo.FooID,
BazInfoProvider.Get()
.InCustomRelationshipWithOrder(FooBazInfo.OBJECT_TYPE, BazInfo.OBJECT_TYPE, Foo.FooID, nameof(BazInfo.BazID), nameof(FooBazInfo.FooBazFooID), nameof(FooBazInfo.FooBazBazID))
.ToList()
);
}
// Get Foo Categories
foreach (var Foo in model.Foos)
{
model.FooCategories.Add(Foo.FooID,
CategoryInfoProvider.Get()
.InCustomRelationshipWithOrder(FooCategoryInfo.OBJECT_TYPE, CategoryInfo.OBJECT_TYPE, Foo.FooID, nameof(CategoryInfo.CategoryID), nameof(FooCategoryInfo.FooCategoryID), nameof(FooCategoryInfo.FooCategoryCategoryID))
.ToList()
);
}
return(View(model));
}
private void ProcessProductPagerPage(ProcessQueueItem pqi)
{
if (cancel)
{
return;
}
var wi = (ExtWareInfo)pqi.Item;
var html = PageRetriever.ReadFromServer(pqi.URL);
var doc = CreateDoc(html);
pqi.Processed = true;
var pages = doc.DocumentNode.SelectNodes("//ul[@class='PagingList'][1]/li/a[@href]");
if (pages == null)
{
wi.Page = 1;
lock (this)
{
lstProcessQueue.Add(new ProcessQueueItem
{
Item = wi,
ItemType = ProcessLevels.ProductList,
URL = pqi.URL
});
}
}
else
{
wi.Page = 1;
lock (this)
{
lstProcessQueue.Add(new ProcessQueueItem
{
Item = wi,
ItemType = ProcessLevels.ProductList,
URL = pqi.URL
});
}
foreach (var page in pages)
{
var name = page.InnerTextOrNull();
if (string.IsNullOrWhiteSpace(name))
{
Log.Warn(page, new NullReferenceException());
continue;
}
wi.Page = ParseInt(name);
lock (this)
{
lstProcessQueue.Add(new ProcessQueueItem
{
Item = wi,
ItemType = ProcessLevels.ProductList,
URL = page.AttributeOrNull("href")
});
}
}
}
OnItemLoaded(null);
pqi.Processed = true;
MessagePrinter.PrintMessage("Product pager processed");
StartOrPushPropertiesThread();
}
