public static string GetSiteRoot()
{
string port = System.Web.HttpContext.Current.Request.ServerVariables["SERVER_PORT"];
if (port == null || port == "80" || port == "443")
{
port = "";
}
else
{
port = ":" + port;
}
string protocol = System.Web.HttpContext.Current.Request.ServerVariables["SERVER_PORT_SECURE"];
if (protocol == null || protocol == "0")
{
protocol = "http://";
}
else
{
protocol = "https://";
}
string sOut = protocol + GetDomain.GetDomainFromUrl(context.Request.Url.ToString()) + port + System.Web.HttpContext.Current.Request.ApplicationPath;
if (sOut.EndsWith("/"))
{
sOut = sOut.Substring(0, sOut.Length - 1);
}
return(sOut);
}
protected void DdlSelectLang_SelectedIndexChanged(object sender, EventArgs e)
{
Response.Clear();
string selectedLang = ((DropDownList)sender).SelectedItem.Value;
// Save cookie
ANOProfile.SaveCookies(selectedLang, CurrentUserIP, null);
CustomSiteMapNode csm = new CustomSiteMapNode();
LangaugeSystem ls = new LangaugeSystem();
Page.Culture = selectedLang;
Page.UICulture = selectedLang;
cSiteMapNode homeSiteMapNode = csm.GetSiteMapNode(2, ls.GetLang(selectedLang).LangId);
string getPath = homeSiteMapNode.SiteMapNodePath;
if (homeSiteMapNode.SiteMapNodeRewrittedPath != null)
{
getPath = "http://" + selectedLang.Split('-')[0] + "." + GetDomain.GetDomainFromUrl(Request.Url.ToString()) + VirtualPathUtility.ToAbsolute(homeSiteMapNode.SiteMapNodeRewrittedPath);
}
Response.Redirect(getPath);
Response.End();
}
public void Build(Site site)
{
Site = site;
Url = site.Url;
DocName = @"\" + GetDomain.GetDomainFromUrl(Url) + ".txt";
using (IRepository <Page> db = new Repository <Page>("FirstAppDB"))
{
Pages = db.Find(page => page.SiteId == Site.Id).ToList();
}
Begin();
}
public ParseDomainParams(string Domain, IEnumerable <string> ReplaceInDecription, IEnumerable <string> SkipDecriptionWhenFound,
string[] CurrencySeparators, string decimalSeparator, DescriptionGetKind descriptionGetKind, SearchPriceKind searchPriceKind)
{
this.Domain = Domain;
ShortDomain = GetDomain.GetDomainFromUrl(Domain);
this.ReplaceInDecription = ReplaceInDecription;
this.SkipDecriptionWhenFound = SkipDecriptionWhenFound;
this.CurrencySeparators = CurrencySeparators;
this.DecimalSeparator = decimalSeparator;
this.DescriptionGetKind = descriptionGetKind;
this.SearchPriceKind = searchPriceKind;
var currencies = new Currencies(DecimalSeparator, CurrencySeparators);
CurRegexes = currencies.GetAllCurRegexes();
PriceRegex = currencies.GetPriceRegex();
}
static void Main(string[] args)
{
string[] urls = new[] {"www.google.com", "http://www.google.co.uk/path1/path2 ", "http://google.co.uk/path1/path2 ",
"http://google.com", "http://google.co.in"};
foreach (var item in urls)
{
string url = item;
if (!Regex.IsMatch(item, "^\\w+://"))
url = "http://" + item;
var domain = GetDomain.GetDomainFromUrl(url);
Console.WriteLine("Original : " + item);
Console.WriteLine("URL : " + url);
Console.WriteLine("Domain : " + domain);
Console.WriteLine("Domain Part : " + domain.Substring(0, domain.IndexOf('.')));
Console.WriteLine();
}
}
public void Build(string url)
{
Url = url;
DocName = @"\" + GetDomain.GetDomainFromUrl(Url) + ".txt";
using (IRepository <Page> db = new Repository <Page>("FirstAppDB"))
{
var page = db.Find(p => p.Url == Url).FirstOrDefault();
FirstPage = page;
if (page == null)
{
return;
}
Pages = db.Find(p => p.SiteId == page.SiteId && p.Depth > page.Depth).ToList();
}
Begin();
}
private void AddPorn()
{
log.Info("Try to find the page including: " + FullRequestUri);
lock (SyncObject)
{
foreach (Tuple <String, HashSet <String> > tset in ImageSrcSetList)
{
if (tset.Item2.Contains(FullRequestUri.ToLower()))
{
string domainName = GetDomain.GetDomainFromUrl(tset.Item1);
PornDatabase.DomainType dt = PornDB.GetDomainType(domainName);
//hit
log.Info("Hit: " + FullRequestUri + " => " + tset.Item1);
//add to database
PornDB.InsertPornPage(domainName, tset.Item1,
FullRequestUri, dt == PornDatabase.DomainType.Black);
}
}
}
}
protected override void OnReceiveRequest()
{
//store complete request uri first, it will parsed and the host will be removed later
FullRequestUri = RequestLine.URI;
DomainName = GetDomain.GetDomainFromUrl(RequestLine.URI);
DomainType = PornDB.GetDomainType(DomainName);
_autoDomainStatus = PornDB.GetAutoDomainStatus(DomainName);
log.Debug(GetDomain.GetDomainFromUrl(RequestLine.URI));
//DomainType == PornDatabase.DomainType.Black ||
if (DomainType != PornDatabase.DomainType.White &&
_autoDomainStatus == PornDatabase.ForbiddenItemStatus.Normal &&
Properties.Settings.Default.IsPornWebsiteProtectionTurnOn)
{
PornDB.InsertBlockedPage(FullRequestUri);
//SocketBP.Send403();
//SocketBP.SendMasaHttpForbidden("I Love You!");
SocketBP.SendMasaHttpForbidden(Properties.Settings.Default.forbiddenRedirectUrl, "<h1>该网页包含不良信息，已被" + Properties.Resources.ProductionName + "屏蔽！</h1>");
//SocketBP.SendHttpError();
State.NextStep = AbortRequest;
log.Info("Block request in tmp black list: " + RequestLine.URI);
}
}
public void Start(Site site, int numberOfThreads)
{
MainSite = site;
NumberOfThreads = numberOfThreads;
MainUrl = new Uri(site.Url);
Domain = GetDomain.GetDomainFromUrl(MainUrl.OriginalString);
PageQueue = new ConcurrentQueue <Page>();
LinksUrl = new ConcurrentHashSet <string>();
Tasks = new Task[numberOfThreads];
ParsePage(new Page {
Depth = 0, Url = MainUrl.OriginalString
});
for (var i = 0; i < NumberOfThreads; i++)
{
Tasks[i] = new Task(Parsing);
Tasks[i].Start();
}
}
protected void Page_Load(object sender, EventArgs e)
{
if (!Page.IsPostBack)
{
try
{
ImgLoading.ImageUrl = ImgLoading.ImageUrl.ChangeToImageHost();
H1Results.InnerHtml += " - " + Request.QueryString["sg"]; // insert game name
#region Search engine
HttpRequest currentR = Request;
// Set values of current lang
LangaugeSystem ls = new LangaugeSystem();
CurrentLang = ls.CurrentLang;
CurrentLangId = ls.CurrentLangId;
#region Get QueryStringsDatas (Convert to htmlencodes)
var searchQueryDatas = new
{
ClanSkill = currentR.QueryString["cs"],
ClanContinent = currentR.QueryString["cct"],
ClanCountry = currentR.QueryString["cc"],
SearchSkill = currentR.QueryString["ss"],
SearchContinent = currentR.QueryString["sct"],
SearchCountry = (string)currentR.QueryString["sc"],
SearchGame = (string)currentR.QueryString["sg"],
SearchGameMode = (string)currentR.QueryString["sgt"]
}.ToAnonymousObjectCollection();
foreach (AnonymousObject s in searchQueryDatas)
{
s.SetValue(Server.HtmlEncode(Server.UrlDecode(s.GetValue <string>())));
}
#endregion
#region Get QueryStrings custom text data (no validate or convert)
var searchQueryCustomTextData = new
{
ClanName = Server.HtmlEncode(Server.UrlDecode(currentR.QueryString["cn"])),
SearchMap = Server.HtmlEncode(Server.UrlDecode(currentR.QueryString["sm"]))
};
#endregion
#region Get QueryStrings customdata
var searchQueryCustomData = new
{
SearchMatchStart = currentR.QueryString["sfd"],
SearchXvs = currentR.QueryString["sxv"],
SearchvsX = currentR.QueryString["svx"]
};
#endregion
try
{
#region Get IDs of searchQueryDatas
int? clanSkillId = null;
SkillsSystem ss = new SkillsSystem();
if (!string.IsNullOrEmpty(searchQueryDatas.GetAnonymousObject("ClanSkill").GetValue <string>()))
{
clanSkillId =
((dynamic)ss.GetSkill(searchQueryDatas.GetAnonymousObject("ClanSkill").GetValue <string>(),
CurrentLangId)).SearchWarSkillId;
}
int?searchSkillId = null;
if (!string.IsNullOrEmpty(searchQueryDatas.GetAnonymousObject("SearchSkill").GetValue <string>()))
{
searchSkillId =
((dynamic)ss.GetSkill(searchQueryDatas.GetAnonymousObject("SearchSkill").GetValue <string>(),
CurrentLangId)).SearchWarSkillId;
}
ContinentSystem cs = new ContinentSystem();
int clanContinentId = ((dynamic)cs.GetContinent(CurrentLangId,
searchQueryDatas.GetAnonymousObject("ClanContinent").
GetValue <string>())).SearchWarContinentId;
CountrySystem cys = new CountrySystem();
int clanCountryId = ((dynamic)cys.GetCountry(CurrentLangId,
searchQueryDatas.GetAnonymousObject("ClanCountry").GetValue
<string>())).SearchWarCountryId;
int searchContinent = ((dynamic)cs.GetContinent(CurrentLangId,
searchQueryDatas.GetAnonymousObject("SearchContinent")
.
GetValue <string>())).SearchWarContinentId;
int?searchCountryId = null;
if (!string.IsNullOrEmpty(searchQueryDatas.GetAnonymousObject("SearchCountry").GetValue <string>()))
{
searchCountryId = ((dynamic)cys.GetCountry(CurrentLangId,
searchQueryDatas.GetAnonymousObject("SearchCountry").
GetValue
<string>())).SearchWarCountryId;
}
GamesSystem gs = new GamesSystem();
int searchGameId = gs.GetGame(searchQueryDatas.GetAnonymousObject("SearchGame").GetValue
<string>()).SearchWarGameId;
int?searchGameModeId = null;
if (!string.IsNullOrEmpty(searchQueryDatas.GetAnonymousObject("SearchGameMode").GetValue <string>()))
{
GameModeSystem gms = new GameModeSystem();
searchGameModeId =
gms.GetGameType(searchQueryDatas.GetAnonymousObject("SearchGameMode").GetValue
<string>()).SearchWarGameTypeId;
}
#endregion
#region Create url to the "client javascript" (get "data search results" as xml)
ANO_User getprofile = ANOProfile.GetCookieValues(CurrentUserIP);
Guid newID = Guid.NewGuid();
ANOProfile.SaveCookies(CurrentUserIP, newID);
xmlSearchUrl = "http://" + GetDomain.GetDomainFromUrl(Context.Request.Url.ToString()) + "/m.ashx";
xmlSearchUrlaction = xmlSearchUrl + "?li= " + CurrentLangId.ToString()
+ "&cn=" + Server.UrlEncode(searchQueryCustomTextData.ClanName)
+ ((clanSkillId.HasValue) ? "&cs=" + clanSkillId.Value.ToString() : "")
+ "&cct=" + clanContinentId.ToString()
+ "&cc=" + clanCountryId.ToString()
+ "&sg=" + searchGameId.ToString()
+ ((searchGameModeId.HasValue) ? "&sgt=" + searchGameModeId.Value.ToString() : "")
+ ((searchSkillId.HasValue) ? "&ss=" + searchSkillId.Value.ToString() : "")
+ "&sct=" + searchContinent.ToString()
+ ((searchCountryId.HasValue) ? "&sc=" + searchCountryId.Value.ToString() : "")
+ "&sxv=" + searchQueryCustomData.SearchXvs
+ "&svx=" + searchQueryCustomData.SearchvsX
+
(!string.IsNullOrEmpty(searchQueryCustomTextData.SearchMap)
? "&sm=" + Server.UrlEncode(searchQueryCustomTextData.SearchMap)
: "")
+ "&sfd=" + Server.UrlEncode(searchQueryCustomData.SearchMatchStart) + "&option=search,chat&action=1";
xmlSearchUrl = xmlSearchUrl + "?option=search,chat&li= " + CurrentLangId.ToString();
#endregion
}
catch
{
// nothing "error"
}
}
catch
{
}
#endregion
}
}
//TODO: Add parsing using schema.org
/// <summary>
/// Wrap a single page and recursively go to all incoming links
/// </summary>
/// <param name="url"></param>
private void Wrap(string url)
{
if (products.Count >= MaxProducts)
{
return;
}
Debug.WriteLine("parsing page:" + url);
var htmlDoc = web.Load(url);
foreach (var prod in htmlDoc.GetProducts(pdp))
{
if (AddIfNotEqual(products, prod))
{
if (products.Count >= MaxProducts)
{
return;
}
}
}
var linkedPages = htmlDoc.DocumentNode.Descendants("a")
.Select(a => a.GetAttributeValue("href", null));
foreach (var s in linkedPages)
{
if (String.IsNullOrEmpty(s) || StringUtils.StringConstants.linksToSkip.Where(x => x.Match(s).Success).Count() > 0)
{
continue;
}
Uri uri;
string domain;
var link = s;
if (!Uri.TryCreate(link, UriKind.Absolute, out uri))
{
domain = pdp.ShortDomain;
link = pdp.Domain + link;
}
else
{
domain = GetDomain.GetDomainFromUrl(uri);
if (domain != pdp.ShortDomain)
{
continue;
}
}
//processing links like http://mysite.com/#somedata
link = link.Split('#')[0];
if (links.Contains(link))
{
continue;
}
links.Add(link);
//TODO: Change this code to:
//Task t= new Task(()=>Wrap(link));
//t.Start();
//tasks.Add(t);
//change collections to asynchronous
Wrap(link);
if (products.Count >= MaxProducts)
{
return;
}
}
//TODO: add Task.WaitAll(tasks);
}
public static void SaveCookies(string shortlang,
Guid?searchMatchID,
string userIpAddress,
HttpContext context,
string timeZone,
Guid?userID)
{
if (context == null)
{
context = HttpContext.Current;
}
// Get UserIpAddress
string getUserIp = userIpAddress;
// Get Cookie
HttpCookie getCookie = context.Request.Cookies[cookieName];
// Set values
HttpCookie createCookie = new HttpCookie(cookieName);
createCookie.Values.Add("ShortLang", shortlang.ToString());
createCookie.Values.Add("IPAddress", getUserIp.EncodeToMd5());
if (context.Request.Cookies.Get(cookieName) == null)
{
if (userID.HasValue)
{
createCookie.Values.Add("UserId", userID.ToString());
}
else
{
createCookie.Values.Add("UserId", Guid.NewGuid().ToString());
}
createCookie.Values.Add("DateAdded", DateTime.UtcNow.Ticks.ToString());
}
else
{
createCookie.Values.Add("DateAdded", getCookie.Values["DateAdded"]);
if (userID.HasValue)
{
createCookie.Values.Add("UserId", userID.ToString());
}
else
{
createCookie.Values.Add("UserId", getCookie.Values["UserId"]);
}
}
if (searchMatchID.HasValue)
{
createCookie.Values.Add("SearchMatchID", searchMatchID.ToString());
}
else
{
if (context.Request.Cookies.Get(cookieName) != null)
{
if (!string.IsNullOrEmpty(getCookie.Values["SearchMatchID"]))
{
createCookie.Values.Add("SearchMatchID", getCookie.Values["SearchMatchID"]);
}
}
}
if (!string.IsNullOrEmpty(timeZone))
{
createCookie.Values.Add("TimeZone", HttpUtility.UrlEncode(timeZone));
}
else
{
if (context.Request.Cookies.Get(cookieName) != null)
{
if (!string.IsNullOrEmpty(getCookie.Values["TimeZone"]))
{
createCookie.Values.Add("TimeZone", getCookie.Values["TimeZone"]);
}
}
}
createCookie.Path = "/";
createCookie.Secure = false;
createCookie.HttpOnly = true;
createCookie.Domain = GetDomain.GetDomainFromUrl(context.Request.Url.ToString());
// Create or Set cookie
if (context.Request.Cookies.Get(cookieName) != null)
{
context.Response.Cookies.Set(createCookie);
}
else
{
context.Response.Cookies.Add(createCookie);
}
}