//public string[] GetAHrefUrls(string data)
//{
//}
public HashSet <string> GetAHrefUrls(string page, Uri pageUri)
{
if (page == null)
{
return(null);
}
HashSet <string> urls = new HashSet <string>();
int founUrls = 0;
MatchCollection curUrls = A_Href_Regex.Matches(page);
foreach (Match m in curUrls)
{
string parsedUri = m.Groups["uri"].Value;
Uri uri = UriHandler.CreateUri(parsedUri, pageUri.Scheme, pageUri.Host);
if (uri != null)
{
if (uri.Host == pageUri.Host && !urls.Contains(uri.OriginalString))
{
urls.Add(uri.OriginalString);
founUrls++;
}
}
}
return(urls);
}
public static void SaveCookies(Uri uri, CookieCollection cookies)
{
if (cookies == null)
{
return;
}
Uri host = UriHandler.CreateUri(string.Empty, uri.Scheme, uri.Host);
if (uri != null)
{
_sharedCookies.Add(host, cookies);
}
}
private static string Parse(string address, string pattern)
{
Uri uri = UriHandler.CreateUri(address);
DownloaderObj obj = new DownloaderObj(uri, null, true, null, CookieOptions.UseShared & CookieOptions.SaveShared, 5);
Downloader.DownloadSync(obj);
if (obj.DataStr != null)
{
return((new Regex(pattern)).Match(obj.DataStr).Groups[1].ToString());
}
else
{
return(null);
}
}
private void SaveCookies(Uri uri, CookieCollection cookies)
{
if (CollectCookie)
{
if (cookies == null || uri == null)
{
return;
}
var host = UriHandler.CreateUri(string.Empty, uri.Scheme, uri.Host);
if (host != null)
{
AllReceivedCookies.Add(host, cookies);
}
}
}
public void ParseProxyFromUrlsOrSerp(List <string> uriList)
{
List <Uri> validUris = new List <Uri>();
foreach (var uri in uriList)
{
Uri validUri = UriHandler.CreateUri(uri);
if (validUri != null)
{
validUris.Add(validUri);
}
}
if (validUris.Count == 0)
{
validUris = null;
}
ParseProxyFromUrlsOrSerp(validUris);
}
bool CheckRedirect(HttpWebResponse response, ref Uri redirUri)
{
if (response.Headers == null)
{
return(false);
}
var location = response.Headers[HttpResponseHeader.Location];
if (location != null)
{
var host = response.ResponseUri.Host;
var sheme = response.ResponseUri.Scheme;
redirUri = UriHandler.CreateUri(location, sheme, host);
return(true);
}
return(false);
}
public PatternsContainer LoadPagePattern(string uristr, string path)
{
List <string> patterns = new List <string>();
try
{
lock (sync)
{
using (StreamReader sw = new StreamReader(path, Encoding.UTF8))
{
while (!sw.EndOfStream)
{
string[] uriAndPatt = sw.ReadLine().Split();
if (uriAndPatt.Length < 2)
{
continue;
}
string line = sw.ReadLine();
if (uriAndPatt[0] == uristr)
{
for (int i = 1; i < uriAndPatt.Length; i++)
{
patterns.Add(uriAndPatt[i]);
}
}
}
}
}
Uri uri = UriHandler.CreateUri(uristr);
if (uri == null)
{
return(null);
}
return(new PatternsContainer(uri, patterns));
}
catch (Exception)
{
return(null);
}
}
bool CheckRedirect(DownloaderObj obj, ref Uri redirUri)
{
if (obj.Response.Headers == null)
{
return(false);
}
string location = obj.Response.Headers[HttpResponseHeader.Location];
if (location != null)
{
string host = obj.Response.ResponseUri.Host;
string sheme = obj.Response.ResponseUri.Scheme;
redirUri = UriHandler.CreateUri(location, sheme, host);
return(true);
}
return(false);
}
public static List <PatternsContainer> LoadPatterns(string path)
{
List <PatternsContainer> paterns = new List <PatternsContainer>();
try
{
lock (sync)
{
using (StreamReader sw = new StreamReader(path, Encoding.UTF8))
{
while (!sw.EndOfStream)
{
string[] uriAndPatt = sw.ReadLine().Split('|');
if (uriAndPatt.Length < 2)
{
continue;
}
Uri uri = UriHandler.CreateUri(uriAndPatt[0]);
if (uri == null)
{
continue;
}
List <string> pattList = new List <string>();
for (int i = 1; i < uriAndPatt.Length; i++)
{
pattList.Add(uriAndPatt[i]);
}
paterns.Add(new PatternsContainer(uri, pattList));
}
}
}
return(paterns);
}
catch (Exception)
{
return(null);
}
}