public void From(Gateway gateway)
{
this.BufferSize = gateway.BufferSize;
this.PoolMaxSize = gateway.PoolMaxSize;
foreach (var server in gateway.Agents.Servers)
{
Servers.Add(new ServerInfo {
MaxConnections = server.MaxConnections, Uri = server.Uri.ToString(), Remark = server.Remark, Category = server.Category
});
}
this.OutputServerAddress = gateway.OutputServerAddress;
this.AgentMaxConnection = gateway.AgentMaxConnection;
this.AgentRequestQueueSize = gateway.AgentRequestQueueSize;
this.GatewayQueueSize = gateway.GatewayQueueSize;
this.InstanceID = gateway.InstanceID;
UrlConfig urlConfig = new UrlConfig();
urlConfig.From(gateway.Routes.Default);
Urls.Add(urlConfig);
foreach (var route in gateway.Routes.Urls)
{
urlConfig = new UrlConfig();
urlConfig.From(route);
Urls.Add(urlConfig);
}
this.PluginConfig = new PluginConfig(gateway.Pluginer);
this.PluginsStatus = gateway.PluginCenter.PluginsStatus;
}
/// <summary>
/// Initializes a new instance of the MainViewModel class.
/// </summary>
public MainViewModel(IDataService dataService)
{
_dataService = dataService;
AddUrlCommand = new RelayCommand(() =>
{
Urls.Add(URL);
URL = string.Empty;
MessageBox.Show("URL added");
});
StartRequestsCommand = new RelayCommand(() =>
{
var ds = (_dataService as DataService);
ds.SetCallback(observable =>
{
_onNextSubscription = observable.ObserveOn(DispatcherScheduler.Current).
Subscribe(resp => Responses.Add(resp),
ex =>
{
MessageBox.Show(ex.Message);
MessageBox.Show("Completed with error");
Cleanup();
},
() =>
{
MessageBox.Show("Completed");
Cleanup();
});
});
ds.GetData(_urls, null);
});
}
public ProxyCrawler_66Ip()
{
for (int i = 1; i <= 10; ++i)
{
Urls.Add(string.Format(@"http://www.66ip.cn/{0}.html", i));
}
}
public bool ForceUrl()
{
if (!string.IsNullOrEmpty(m_websiteString))
{
Urls.Add(m_websiteString);
SetThirdLevelRequest();
return(true);
}
return(false);
}
private Task RunHost(CancellationToken token)
{
try
{
var appLifetime = (IApplicationLifetime)_host.Services.GetService(typeof(IApplicationLifetime));
appLifetime.ApplicationStarted.Register(() =>
{
var addresses = _host.ServerFeatures
.Get <Microsoft.AspNetCore.Hosting.Server.Features.IServerAddressesFeature>()
.Addresses;
foreach (string address in addresses)
{
Urls.Add(address.Replace("0.0.0.0", "localhost").Replace("[::]", "localhost"));
PortUtils.TryExtract(address, out bool isHttps, out string protocol, out string host, out int port);
Ports.Add(port);
}
IsStarted = true;
});
#if NETSTANDARD1_3
_logger.Info("WireMock.Net server using netstandard1.3");
#elif NETSTANDARD2_0
_logger.Info("WireMock.Net server using netstandard2.0");
#elif NETSTANDARD2_1
_logger.Info("WireMock.Net server using netstandard2.1");
#elif NETCOREAPP3_1
_logger.Info("WireMock.Net server using .NET Core 3.1");
#elif NET5_0
_logger.Info("WireMock.Net server using .NET 5.0");
#elif NET46
_logger.Info("WireMock.Net server using .NET Framework 4.6.1 or higher");
#endif
#if NETSTANDARD1_3
return(Task.Run(() =>
{
_host.Run(token);
}));
#else
return(_host.RunAsync(token));
#endif
}
catch (Exception e)
{
_runningException = e;
_logger.Error(e.ToString());
IsStarted = false;
return(Task.CompletedTask);
}
}
public void SetUrl(string site, string url)
{
if (Urls.ContainsKey(site))
{
Urls[site] = url;
}
else
{
Urls.Add(site, url);
}
}
private void ParseUrls(System.Xml.XmlReader reader)
{
reader.Read();
while (reader.LocalName == "url")
{
PhotoInfoUrl url = new PhotoInfoUrl();
((IFlickrParsable)url).Load(reader);
Urls.Add(url);
}
}
private void Save(HtmlDocument doc)
{
try
{
doc.Save(FullPath);
Urls.Add(FileName);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
public override void Initialize()
{
Urls.Add(Constraints.BackEndProxyTLSUrl);
Urls.Add(Constraints.EstablishTrustUrl);
Urls.Add(Constraints.GetSTSConfigurationUrl);
Urls.Add(Constraints.ProxyTrustUrl);
Urls.Add(Constraints.RelyingPartyTrustUrl);
Urls.Add(Constraints.RenewTrustUrl);
Urls.Add(Constraints.StoreUrl);
Urls.Add(Constraints.FederationAuthUrl);
base.Initialize();
}
/// <summary>
/// 添加URL列表
/// </summary>
/// <param name="urls">URL列表</param>
public void AddUrls(IList <string> urls)
{
if (urls != null)
{
foreach (var u in urls)
{
if (!Urls.Contains(u))
{
Urls.Add(u);
}
}
}
}
public OwinSelfHost([NotNull] WireMockMiddlewareOptions options, [NotNull] params string[] uriPrefixes)
{
Check.NotNull(options, nameof(options));
Check.NotEmpty(uriPrefixes, nameof(uriPrefixes));
foreach (string uriPrefix in uriPrefixes)
{
var uri = new Uri(uriPrefix);
Urls.Add(uri);
Ports.Add(uri.Port);
}
_options = options;
}
public void AddSelectionDelegate(object sender)
{
var youtubeResult = (YoutubeResult)sender;
Console.WriteLine("result that's being added = {0}", youtubeResult.Title);
if (Urls.Any(url => url.Item1 == youtubeResult.Url))
{
return;
}
var tup = Tuple.Create(youtubeResult.Url, youtubeResult.Title);
Urls.Add(tup);
}
/// <inheritdoc />
public override void ReadRaw(byte[] raw)
{
if (raw.Length == 0)
{
throw new OptionLengthZeroException();
}
string tmp = Encoding.UTF8.GetString(raw, 0, raw.Length);
foreach (string url in tmp.Split(' '))
{
Urls.Add(url);
}
}
public OwinSelfHost([NotNull] IWireMockMiddlewareOptions options, [NotNull] HostUrlOptions urlOptions)
{
Check.NotNull(options, nameof(options));
Check.NotNull(urlOptions, nameof(urlOptions));
_options = options;
_logger = options.Logger ?? new WireMockConsoleLogger();
foreach (var detail in urlOptions.GetDetails())
{
Urls.Add(detail.Url);
Ports.Add(detail.Port);
}
}
IFluentRouteConfiguration IFluentRouteConfiguration.Url(string culture, string url)
{
if (culture == null)
{
throw new ArgumentNullException("culture");
}
if (url == null)
{
throw new ArgumentNullException("url");
}
Urls.Add(culture, url);
return(this);
}
public OwinSelfHost([NotNull] WireMockMiddlewareOptions options, [NotNull] params string[] uriPrefixes)
{
Check.NotNull(options, nameof(options));
Check.NotNullOrEmpty(uriPrefixes, nameof(uriPrefixes));
foreach (string uriPrefix in uriPrefixes)
{
Urls.Add(uriPrefix);
PortUtil.TryExtractProtocolAndPort(uriPrefix, out string host, out int port);
Ports.Add(port);
}
_options = options;
}
public ProxyCrawler_Xici()
{
for (int i = 1; i <= 10; ++i)
{
Urls.Add("http://www.xicidaili.com/nn/" + i);
}
for (int i = 1; i <= 10; ++i)
{
Urls.Add("http://www.xicidaili.com/nt/" + i);
}
for (int i = 1; i <= 10; ++i)
{
Urls.Add("http://www.xicidaili.com/wt/" + i);
}
}
private void LoadPage(string url, ufFormatDescriber format)
{
if (url != string.Empty)
{
Url urlReport = new Url();
urlReport.Address = url;
//m_oFormatDescriber = format;
UrlModule urlModule = new UrlModule();
Uri uri = new Uri(url);
urlModule.DocumentContentType = UrlModule.ContentType.Html;
urlModule.DocumentRequestType = UrlModule.RequestType.Get;
urlModule.Load(uri);
urlReport.Status = urlModule.StatusCode;
DateTime started = DateTime.Now;
m_AllParsedUrls.Add(urlReport);
if (urlModule.StatusCode == 200 && urlModule.Html != null)
{
ParseUf(urlModule.Html, uri.ToString(), format);
}
//----------------------
Urls newUrls = new Urls();
m_oMe.Nodes.Add(m_oData);
foreach (ufDateNode node in m_oData.Nodes)
{
if (node.Nodes["link"] != null)
{
string link = node.Nodes["link"].Value;
if (ShouldParseUrl(link))
{
newUrls.Add(new Url(link));
}
}
}
foreach (Url newUrl in newUrls)
{
LoadPage(newUrl.Address, m_oMeDescriber);
}
//----------------------
DateTime ended = DateTime.Now;
urlReport.LoadTime = ended.Subtract(started);
}
}
public ShortUrl ShortenUrl(string fullUrl)
{
ShortUrl newUrl = new ShortUrl()
{
Full = fullUrl, Time = DateTime.Now
};
string keyString = StringExtensions.GetRandomString();
while (CheckIfKeyExist(keyString))
{
keyString = StringExtensions.GetRandomString();
}
newUrl.Short = keyString;
Urls.Add(newUrl);
return(newUrl);
}
private void ImportFile(string fileName, bool additional = false)
{
if (!string.IsNullOrEmpty(fileName))
{
Urls.Clear();
bwImportFile = new BackgroundWorker();
bwImportFile.DoWork += (s, e) =>
{
string cfileName = e.Argument as string;
string[] res = Helper.GetLinksFromFile(cfileName, additional, true).OrderBy(i => i).ToArray();
e.Result = res;
};
bwImportFile.RunWorkerCompleted += (s, e) =>
{
try
{
if (e.Error != null)
{
throw e.Error;
}
string[] res = (string[])e.Result;
foreach (var sw in res.Select(i => new StringUrlWithResultWrapper()
{
Value = i
}))
{
Urls.Add(sw);
}
}
catch (Exception ex)
{
MessageBox.Show(string.Format("Ошибка при импорте файла:{0}{1}", Environment.NewLine, ex.Message), "Ошибка", MessageBoxButton.OK, MessageBoxImage.Error);
}
finally
{
IsBusy = false;
bwImportFile.Dispose();
bwImportFile = null;
}
};
IsBusy = true;
bwImportFile.RunWorkerAsync(fileName);
}
}
public OwinSelfHost([NotNull] IWireMockMiddlewareOptions options, [NotNull] params string[] uriPrefixes)
{
Check.NotNull(options, nameof(options));
Check.NotNullOrEmpty(uriPrefixes, nameof(uriPrefixes));
_logger = options.Logger ?? new WireMockConsoleLogger();
foreach (string uriPrefix in uriPrefixes)
{
Urls.Add(uriPrefix);
PortUtils.TryExtract(uriPrefix, out string protocol, out string host, out int port);
Ports.Add(port);
}
_options = options;
}
//启动
public void Start()
{
try
{
Urls.Add(StartUrl, false);
}
catch (ArgumentException e)
{
Inform(this, new InformEventArgs()
{
Url = StartUrl, Message = "已存在"
});
}
Inform(this, new InformEventArgs()
{
Url = null, Message = "开始爬取"
});
while (true)
{
string currentUrl = null;
foreach (string url in Urls.Keys)
{
if ((bool)Urls[url])
{
continue;
}
currentUrl = url;
}
if (currentUrl == null || Count > 10)
{
break;
}
string html = Download(currentUrl);
Urls[currentUrl] = true;
Count++;
Parse(html, currentUrl);
}
Inform(this, new InformEventArgs()
{
Url = null, Message = "结束爬取"
});
}
private void AddUrls()
{
foreach (string url in configuration.Url)
{
if (Urls.Contains(url))
{
Logger.WriteLine($"Duplicate Url found: {url}");
}
else if (!IsProperFormatGithubUrl(url))
{
Logger.WriteLine($"Invalid Url found: {url}");
}
else
{
Logger.WriteLine($"Added {url} to Urls.");
Urls.Add(url);
}
}
}
private void ParseUrl(KeyValuePair <string, int> url)
{
HtmlDocument doc = new HtmlDocument();
try
{
doc.LoadHtml(HttpGet.Get(url.Key).Content.ReadAsStringAsync().Result);
}
catch (Exception)
{
}
HtmlNodeCollection hrefs = doc.DocumentNode.SelectNodes(@"//a[@href and not(contains(@href, 'javascript'))]");
if (hrefs == null)
{
return;
}
foreach (HtmlNode nodeA in hrefs)
{
var Ref = nodeA.Attributes["href"].Value.Trim();
if (!Ref.StartsWith(Url.ToString()))
{
Uri newUri = null;
if (Uri.TryCreate(Url, Ref, out newUri))
{
Ref = newUri.ToString();
}
}
if (!Urls.Contains(Ref))
{
Urls.Add(Ref);
_To.OnNewUrl(Ref);
if (url.Value < MaxTire)
{
OnNewUrl(Ref, url.Value + 1);
}
}
}
}
public void From(Gateway gateway)
{
foreach (var server in gateway.Agents.Servers)
{
Servers.Add(new ServerInfo {
MaxConnections = server.MaxConnections, Uri = server.Uri.ToString()
});
}
UrlConfig urlConfig = new UrlConfig();
urlConfig.From(gateway.Routes.Default);
Urls.Add(urlConfig);
foreach (var route in gateway.Routes.Urls)
{
urlConfig = new UrlConfig();
urlConfig.From(route);
Urls.Add(urlConfig);
}
this.PluginConfig = new PluginConfig(gateway.Pluginer);
}
private void InvokerProc(string url)
{
HtmlDocument doc = new HtmlDocument();
try
{
doc.LoadHtml(HttpGet.Get(url).Content.ReadAsStringAsync().Result);
}
catch (Exception)
{
// todo
}
var answersSet = ReadAllXPathNode(doc, AnsersPath);
var asnwerSet = ReadAllXPathNode(doc, AnserPath);
if (answersSet != null)
{
foreach (var answers in answersSet)
{
if (!Urls.Contains(answers))
{
Urls.Add(answers);
OnNewUrl(answers);
}
}
}
if (asnwerSet != null)
{
foreach (var answer in asnwerSet)
{
if (!Urls.Contains(answer))
{
Urls.Add(answer);
_To.OnNewUrl(answer);
}
}
}
}
public override void ReadFrom(XElement xE)
{
base.ReadFrom(xE);
Urls = null;
IncludeSubUrls = null;
foreach (var xItem in xE.Elements())
{
var localName = xItem.Name.LocalName;
if (localName == "urls")
{
if (Urls == null)
{
Urls = new List <string>();
}
Urls.Add(xItem.Value);
}
else if (localName == "includeSubUrls")
{
IncludeSubUrls = bool.Parse(xItem.Value);
}
}
}
public override bool FilterValidUrls()
{
string sort = m_parameterData.GetParameterValue("sort");
string limitStr = m_parameterData.GetParameterValue("limit");
if (!int.TryParse(limitStr, out int limit) || limit > Constants.MAX_LIMIT)
{
limit = Constants.DEFAULT_LIMIT;
}
string offsetStr = m_parameterData.GetParameterValue("offset");
int.TryParse(offsetStr, out int offset);
int lowerLimit = (offset - 1) / 20;
int remainder = 0;
if ((offset - 1) % 20 != 0)
{
remainder = 20 % ((offset - 1) % 20);
}
int upperLimit = lowerLimit + (limit - 1 - remainder) / 20;
if (offset - 1 + limit > 20 * (upperLimit + 1))
{
upperLimit++;
}
for (int idx = lowerLimit; idx <= upperLimit; ++idx)
{
Urls.Add(string.Format(@"search/{0}/{1}/results?page={2}&sort={3}",
GetThirdLevelRequest(), SearchString.Replace(" ", "%20"), idx, sort));
}
return(Urls.Count > 0);
}
private void Parse(string html, string previous)
{
Uri uri = new Uri(previous);
foreach (Match match in HREF_REGEX.Matches(html))
{
string url = match.Groups["url"].Value;
if (!URL_REGEX.IsMatch(url))
{
continue;
}
Uri add = new Uri(url, UriKind.RelativeOrAbsolute);
string result;
if (url.StartsWith("http://") || url.StartsWith("https://"))
{
if (add.Host != uri.Host)
{
continue;
}
result = add.AbsoluteUri;
}
else if (!add.IsAbsoluteUri)
{
result = new Uri(uri, add).AbsoluteUri;
}
else
{
continue;
}
if (!Urls.Contains(result) && !CompletedUrls.Contains(result) && !DownloadingUrls.Contains(result))
{
Urls.Add(result);
}
}
}
public virtual void ReadFrom(XElement xE)
{
MediaId = null;
Type = null;
ReferenceId = null;
Dimensions = null;
Urls = null;
MimeType = null;
SourceUrl = null;
Name = null;
FileSize = null;
CreationTime = null;
MediaType = null;
foreach (var xItem in xE.Elements())
{
var localName = xItem.Name.LocalName;
if (localName == "mediaId")
{
MediaId = long.Parse(xItem.Value);
}
else if (localName == "type")
{
Type = MediaMediaTypeExtensions.Parse(xItem.Value);
}
else if (localName == "referenceId")
{
ReferenceId = long.Parse(xItem.Value);
}
else if (localName == "dimensions")
{
if (Dimensions == null)
{
Dimensions = new List <Media_Size_DimensionsMapEntry>();
}
var dimensionsItem = new Media_Size_DimensionsMapEntry();
dimensionsItem.ReadFrom(xItem);
Dimensions.Add(dimensionsItem);
}
else if (localName == "urls")
{
if (Urls == null)
{
Urls = new List <Media_Size_StringMapEntry>();
}
var urlsItem = new Media_Size_StringMapEntry();
urlsItem.ReadFrom(xItem);
Urls.Add(urlsItem);
}
else if (localName == "mimeType")
{
MimeType = MediaMimeTypeExtensions.Parse(xItem.Value);
}
else if (localName == "sourceUrl")
{
SourceUrl = xItem.Value;
}
else if (localName == "name")
{
Name = xItem.Value;
}
else if (localName == "fileSize")
{
FileSize = long.Parse(xItem.Value);
}
else if (localName == "creationTime")
{
CreationTime = xItem.Value;
}
else if (localName == "Media.Type")
{
MediaType = xItem.Value;
}
}
}
private void LoadPage(string url, ufFormatDescriber format)
{
if (url != string.Empty)
{
Url urlReport = new Url();
urlReport.Address = url;
//m_oFormatDescriber = format;
UrlModule urlModule = new UrlModule();
Uri uri = new Uri(url);
urlModule.DocumentContentType = UrlModule.ContentType.Html;
urlModule.DocumentRequestType = UrlModule.RequestType.Get;
urlModule.Load(uri);
urlReport.Status = urlModule.StatusCode;
DateTime started = DateTime.Now;
m_AllParsedUrls.Add(urlReport);
if (urlModule.StatusCode == 200 && urlModule.Html != null)
ParseUf(urlModule.Html, uri.ToString(), format);
//----------------------
Urls newUrls = new Urls();
m_oMe.Nodes.Add(m_oData);
foreach(ufDateNode node in m_oData.Nodes)
{
if (node.Nodes["link"] != null)
{
string link = node.Nodes["link"].Value;
if (ShouldParseUrl(link))
newUrls.Add(new Url(link));
}
}
foreach (Url newUrl in newUrls)
{
LoadPage(newUrl.Address, m_oMeDescriber);
}
//----------------------
DateTime ended = DateTime.Now;
urlReport.LoadTime = ended.Subtract(started);
}
}
