Пример #1
0
        public void From(Gateway gateway)
        {
            this.BufferSize  = gateway.BufferSize;
            this.PoolMaxSize = gateway.PoolMaxSize;
            foreach (var server in gateway.Agents.Servers)
            {
                Servers.Add(new ServerInfo {
                    MaxConnections = server.MaxConnections, Uri = server.Uri.ToString(), Remark = server.Remark, Category = server.Category
                });
            }
            this.OutputServerAddress   = gateway.OutputServerAddress;
            this.AgentMaxConnection    = gateway.AgentMaxConnection;
            this.AgentRequestQueueSize = gateway.AgentRequestQueueSize;
            this.GatewayQueueSize      = gateway.GatewayQueueSize;
            this.InstanceID            = gateway.InstanceID;
            UrlConfig urlConfig = new UrlConfig();

            urlConfig.From(gateway.Routes.Default);
            Urls.Add(urlConfig);
            foreach (var route in gateway.Routes.Urls)
            {
                urlConfig = new UrlConfig();
                urlConfig.From(route);
                Urls.Add(urlConfig);
            }
            this.PluginConfig  = new PluginConfig(gateway.Pluginer);
            this.PluginsStatus = gateway.PluginCenter.PluginsStatus;
        }
Пример #2
0
 /// <summary>
 /// Initializes a new instance of the MainViewModel class.
 /// </summary>
 public MainViewModel(IDataService dataService)
 {
     _dataService  = dataService;
     AddUrlCommand = new RelayCommand(() =>
     {
         Urls.Add(URL);
         URL = string.Empty;
         MessageBox.Show("URL added");
     });
     StartRequestsCommand = new RelayCommand(() =>
     {
         var ds = (_dataService as DataService);
         ds.SetCallback(observable =>
         {
             _onNextSubscription = observable.ObserveOn(DispatcherScheduler.Current).
                                   Subscribe(resp => Responses.Add(resp),
                                             ex =>
             {
                 MessageBox.Show(ex.Message);
                 MessageBox.Show("Completed with error");
                 Cleanup();
             },
                                             () =>
             {
                 MessageBox.Show("Completed");
                 Cleanup();
             });
         });
         ds.GetData(_urls, null);
     });
 }
Пример #3
0
 public ProxyCrawler_66Ip()
 {
     for (int i = 1; i <= 10; ++i)
     {
         Urls.Add(string.Format(@"http://www.66ip.cn/{0}.html", i));
     }
 }
Пример #4
0
 public bool ForceUrl()
 {
     if (!string.IsNullOrEmpty(m_websiteString))
     {
         Urls.Add(m_websiteString);
         SetThirdLevelRequest();
         return(true);
     }
     return(false);
 }
Пример #5
0
        private Task RunHost(CancellationToken token)
        {
            try
            {
                var appLifetime = (IApplicationLifetime)_host.Services.GetService(typeof(IApplicationLifetime));
                appLifetime.ApplicationStarted.Register(() =>
                {
                    var addresses = _host.ServerFeatures
                                    .Get <Microsoft.AspNetCore.Hosting.Server.Features.IServerAddressesFeature>()
                                    .Addresses;

                    foreach (string address in addresses)
                    {
                        Urls.Add(address.Replace("0.0.0.0", "localhost").Replace("[::]", "localhost"));

                        PortUtils.TryExtract(address, out bool isHttps, out string protocol, out string host, out int port);
                        Ports.Add(port);
                    }

                    IsStarted = true;
                });

#if NETSTANDARD1_3
                _logger.Info("WireMock.Net server using netstandard1.3");
#elif NETSTANDARD2_0
                _logger.Info("WireMock.Net server using netstandard2.0");
#elif NETSTANDARD2_1
                _logger.Info("WireMock.Net server using netstandard2.1");
#elif NETCOREAPP3_1
                _logger.Info("WireMock.Net server using .NET Core 3.1");
#elif NET5_0
                _logger.Info("WireMock.Net server using .NET 5.0");
#elif NET46
                _logger.Info("WireMock.Net server using .NET Framework 4.6.1 or higher");
#endif

#if NETSTANDARD1_3
                return(Task.Run(() =>
                {
                    _host.Run(token);
                }));
#else
                return(_host.RunAsync(token));
#endif
            }
            catch (Exception e)
            {
                _runningException = e;
                _logger.Error(e.ToString());

                IsStarted = false;

                return(Task.CompletedTask);
            }
        }
Пример #6
0
 public void SetUrl(string site, string url)
 {
     if (Urls.ContainsKey(site))
     {
         Urls[site] = url;
     }
     else
     {
         Urls.Add(site, url);
     }
 }
Пример #7
0
        private void ParseUrls(System.Xml.XmlReader reader)
        {
            reader.Read();

            while (reader.LocalName == "url")
            {
                PhotoInfoUrl url = new PhotoInfoUrl();
                ((IFlickrParsable)url).Load(reader);
                Urls.Add(url);
            }
        }
Пример #8
0
 private void Save(HtmlDocument doc)
 {
     try
     {
         doc.Save(FullPath);
         Urls.Add(FileName);
     }
     catch (Exception ex)
     {
         Console.WriteLine(ex.Message);
     }
 }
Пример #9
0
 public override void Initialize()
 {
     Urls.Add(Constraints.BackEndProxyTLSUrl);
     Urls.Add(Constraints.EstablishTrustUrl);
     Urls.Add(Constraints.GetSTSConfigurationUrl);
     Urls.Add(Constraints.ProxyTrustUrl);
     Urls.Add(Constraints.RelyingPartyTrustUrl);
     Urls.Add(Constraints.RenewTrustUrl);
     Urls.Add(Constraints.StoreUrl);
     Urls.Add(Constraints.FederationAuthUrl);
     base.Initialize();
 }
Пример #10
0
 /// <summary>
 /// 添加URL列表
 /// </summary>
 /// <param name="urls">URL列表</param>
 public void AddUrls(IList <string> urls)
 {
     if (urls != null)
     {
         foreach (var u in urls)
         {
             if (!Urls.Contains(u))
             {
                 Urls.Add(u);
             }
         }
     }
 }
Пример #11
0
        public OwinSelfHost([NotNull] WireMockMiddlewareOptions options, [NotNull] params string[] uriPrefixes)
        {
            Check.NotNull(options, nameof(options));
            Check.NotEmpty(uriPrefixes, nameof(uriPrefixes));

            foreach (string uriPrefix in uriPrefixes)
            {
                var uri = new Uri(uriPrefix);
                Urls.Add(uri);
                Ports.Add(uri.Port);
            }

            _options = options;
        }
Пример #12
0
        public void AddSelectionDelegate(object sender)
        {
            var youtubeResult = (YoutubeResult)sender;

            Console.WriteLine("result that's being added = {0}", youtubeResult.Title);
            if (Urls.Any(url => url.Item1 == youtubeResult.Url))
            {
                return;
            }

            var tup = Tuple.Create(youtubeResult.Url, youtubeResult.Title);

            Urls.Add(tup);
        }
        /// <inheritdoc />
        public override void ReadRaw(byte[] raw)
        {
            if (raw.Length == 0)
            {
                throw new OptionLengthZeroException();
            }

            string tmp = Encoding.UTF8.GetString(raw, 0, raw.Length);

            foreach (string url in tmp.Split(' '))
            {
                Urls.Add(url);
            }
        }
Пример #14
0
        public OwinSelfHost([NotNull] IWireMockMiddlewareOptions options, [NotNull] HostUrlOptions urlOptions)
        {
            Check.NotNull(options, nameof(options));
            Check.NotNull(urlOptions, nameof(urlOptions));

            _options = options;
            _logger  = options.Logger ?? new WireMockConsoleLogger();

            foreach (var detail in urlOptions.GetDetails())
            {
                Urls.Add(detail.Url);
                Ports.Add(detail.Port);
            }
        }
Пример #15
0
        IFluentRouteConfiguration IFluentRouteConfiguration.Url(string culture, string url)
        {
            if (culture == null)
            {
                throw new ArgumentNullException("culture");
            }
            if (url == null)
            {
                throw new ArgumentNullException("url");
            }

            Urls.Add(culture, url);

            return(this);
        }
Пример #16
0
        public OwinSelfHost([NotNull] WireMockMiddlewareOptions options, [NotNull] params string[] uriPrefixes)
        {
            Check.NotNull(options, nameof(options));
            Check.NotNullOrEmpty(uriPrefixes, nameof(uriPrefixes));

            foreach (string uriPrefix in uriPrefixes)
            {
                Urls.Add(uriPrefix);

                PortUtil.TryExtractProtocolAndPort(uriPrefix, out string host, out int port);
                Ports.Add(port);
            }

            _options = options;
        }
 public ProxyCrawler_Xici()
 {
     for (int i = 1; i <= 10; ++i)
     {
         Urls.Add("http://www.xicidaili.com/nn/" + i);
     }
     for (int i = 1; i <= 10; ++i)
     {
         Urls.Add("http://www.xicidaili.com/nt/" + i);
     }
     for (int i = 1; i <= 10; ++i)
     {
         Urls.Add("http://www.xicidaili.com/wt/" + i);
     }
 }
Пример #18
0
        private void LoadPage(string url, ufFormatDescriber format)
        {
            if (url != string.Empty)
            {
                Url urlReport = new Url();
                urlReport.Address = url;
                //m_oFormatDescriber = format;

                UrlModule urlModule = new UrlModule();
                Uri       uri       = new Uri(url);
                urlModule.DocumentContentType = UrlModule.ContentType.Html;
                urlModule.DocumentRequestType = UrlModule.RequestType.Get;
                urlModule.Load(uri);

                urlReport.Status = urlModule.StatusCode;
                DateTime started = DateTime.Now;
                m_AllParsedUrls.Add(urlReport);

                if (urlModule.StatusCode == 200 && urlModule.Html != null)
                {
                    ParseUf(urlModule.Html, uri.ToString(), format);
                }

                //----------------------
                Urls newUrls = new Urls();
                m_oMe.Nodes.Add(m_oData);

                foreach (ufDateNode node in m_oData.Nodes)
                {
                    if (node.Nodes["link"] != null)
                    {
                        string link = node.Nodes["link"].Value;
                        if (ShouldParseUrl(link))
                        {
                            newUrls.Add(new Url(link));
                        }
                    }
                }
                foreach (Url newUrl in newUrls)
                {
                    LoadPage(newUrl.Address, m_oMeDescriber);
                }
                //----------------------

                DateTime ended = DateTime.Now;
                urlReport.LoadTime = ended.Subtract(started);
            }
        }
Пример #19
0
        public ShortUrl ShortenUrl(string fullUrl)
        {
            ShortUrl newUrl = new ShortUrl()
            {
                Full = fullUrl, Time = DateTime.Now
            };
            string keyString = StringExtensions.GetRandomString();

            while (CheckIfKeyExist(keyString))
            {
                keyString = StringExtensions.GetRandomString();
            }
            newUrl.Short = keyString;
            Urls.Add(newUrl);
            return(newUrl);
        }
        private void ImportFile(string fileName, bool additional = false)
        {
            if (!string.IsNullOrEmpty(fileName))
            {
                Urls.Clear();

                bwImportFile         = new BackgroundWorker();
                bwImportFile.DoWork += (s, e) =>
                {
                    string   cfileName = e.Argument as string;
                    string[] res       = Helper.GetLinksFromFile(cfileName, additional, true).OrderBy(i => i).ToArray();
                    e.Result = res;
                };
                bwImportFile.RunWorkerCompleted += (s, e) =>
                {
                    try
                    {
                        if (e.Error != null)
                        {
                            throw e.Error;
                        }

                        string[] res = (string[])e.Result;
                        foreach (var sw in res.Select(i => new StringUrlWithResultWrapper()
                        {
                            Value = i
                        }))
                        {
                            Urls.Add(sw);
                        }
                    }
                    catch (Exception ex)
                    {
                        MessageBox.Show(string.Format("Ошибка при импорте файла:{0}{1}", Environment.NewLine, ex.Message), "Ошибка", MessageBoxButton.OK, MessageBoxImage.Error);
                    }
                    finally
                    {
                        IsBusy = false;
                        bwImportFile.Dispose();
                        bwImportFile = null;
                    }
                };
                IsBusy = true;
                bwImportFile.RunWorkerAsync(fileName);
            }
        }
Пример #21
0
        public OwinSelfHost([NotNull] IWireMockMiddlewareOptions options, [NotNull] params string[] uriPrefixes)
        {
            Check.NotNull(options, nameof(options));
            Check.NotNullOrEmpty(uriPrefixes, nameof(uriPrefixes));

            _logger = options.Logger ?? new WireMockConsoleLogger();

            foreach (string uriPrefix in uriPrefixes)
            {
                Urls.Add(uriPrefix);

                PortUtils.TryExtract(uriPrefix, out string protocol, out string host, out int port);
                Ports.Add(port);
            }

            _options = options;
        }
Пример #22
0
        //启动
        public void Start()
        {
            try
            {
                Urls.Add(StartUrl, false);
            }
            catch (ArgumentException e)
            {
                Inform(this, new InformEventArgs()
                {
                    Url = StartUrl, Message = "已存在"
                });
            }

            Inform(this, new InformEventArgs()
            {
                Url = null, Message = "开始爬取"
            });
            while (true)
            {
                string currentUrl = null;
                foreach (string url in Urls.Keys)
                {
                    if ((bool)Urls[url])
                    {
                        continue;
                    }
                    currentUrl = url;
                }

                if (currentUrl == null || Count > 10)
                {
                    break;
                }

                string html = Download(currentUrl);
                Urls[currentUrl] = true;
                Count++;
                Parse(html, currentUrl);
            }
            Inform(this, new InformEventArgs()
            {
                Url = null, Message = "结束爬取"
            });
        }
Пример #23
0
 private void AddUrls()
 {
     foreach (string url in configuration.Url)
     {
         if (Urls.Contains(url))
         {
             Logger.WriteLine($"Duplicate Url found: {url}");
         }
         else if (!IsProperFormatGithubUrl(url))
         {
             Logger.WriteLine($"Invalid Url found: {url}");
         }
         else
         {
             Logger.WriteLine($"Added {url} to Urls.");
             Urls.Add(url);
         }
     }
 }
Пример #24
0
        private void ParseUrl(KeyValuePair <string, int> url)
        {
            HtmlDocument doc = new HtmlDocument();

            try
            {
                doc.LoadHtml(HttpGet.Get(url.Key).Content.ReadAsStringAsync().Result);
            }
            catch (Exception)
            {
            }

            HtmlNodeCollection hrefs = doc.DocumentNode.SelectNodes(@"//a[@href and not(contains(@href, 'javascript'))]");

            if (hrefs == null)
            {
                return;
            }

            foreach (HtmlNode nodeA in hrefs)
            {
                var Ref = nodeA.Attributes["href"].Value.Trim();
                if (!Ref.StartsWith(Url.ToString()))
                {
                    Uri newUri = null;
                    if (Uri.TryCreate(Url, Ref, out newUri))
                    {
                        Ref = newUri.ToString();
                    }
                }

                if (!Urls.Contains(Ref))
                {
                    Urls.Add(Ref);
                    _To.OnNewUrl(Ref);

                    if (url.Value < MaxTire)
                    {
                        OnNewUrl(Ref, url.Value + 1);
                    }
                }
            }
        }
Пример #25
0
        public void From(Gateway gateway)
        {
            foreach (var server in gateway.Agents.Servers)
            {
                Servers.Add(new ServerInfo {
                    MaxConnections = server.MaxConnections, Uri = server.Uri.ToString()
                });
            }
            UrlConfig urlConfig = new UrlConfig();

            urlConfig.From(gateway.Routes.Default);
            Urls.Add(urlConfig);
            foreach (var route in gateway.Routes.Urls)
            {
                urlConfig = new UrlConfig();
                urlConfig.From(route);
                Urls.Add(urlConfig);
            }
            this.PluginConfig = new PluginConfig(gateway.Pluginer);
        }
Пример #26
0
        private void InvokerProc(string url)
        {
            HtmlDocument doc = new HtmlDocument();

            try
            {
                doc.LoadHtml(HttpGet.Get(url).Content.ReadAsStringAsync().Result);
            }
            catch (Exception)
            {
                // todo
            }

            var answersSet = ReadAllXPathNode(doc, AnsersPath);
            var asnwerSet  = ReadAllXPathNode(doc, AnserPath);

            if (answersSet != null)
            {
                foreach (var answers in answersSet)
                {
                    if (!Urls.Contains(answers))
                    {
                        Urls.Add(answers);
                        OnNewUrl(answers);
                    }
                }
            }

            if (asnwerSet != null)
            {
                foreach (var answer in asnwerSet)
                {
                    if (!Urls.Contains(answer))
                    {
                        Urls.Add(answer);
                        _To.OnNewUrl(answer);
                    }
                }
            }
        }
 public override void ReadFrom(XElement xE)
 {
     base.ReadFrom(xE);
     Urls           = null;
     IncludeSubUrls = null;
     foreach (var xItem in xE.Elements())
     {
         var localName = xItem.Name.LocalName;
         if (localName == "urls")
         {
             if (Urls == null)
             {
                 Urls = new List <string>();
             }
             Urls.Add(xItem.Value);
         }
         else if (localName == "includeSubUrls")
         {
             IncludeSubUrls = bool.Parse(xItem.Value);
         }
     }
 }
Пример #28
0
        public override bool FilterValidUrls()
        {
            string sort = m_parameterData.GetParameterValue("sort");

            string limitStr = m_parameterData.GetParameterValue("limit");

            if (!int.TryParse(limitStr, out int limit) || limit > Constants.MAX_LIMIT)
            {
                limit = Constants.DEFAULT_LIMIT;
            }

            string offsetStr = m_parameterData.GetParameterValue("offset");

            int.TryParse(offsetStr, out int offset);

            int lowerLimit = (offset - 1) / 20;
            int remainder  = 0;

            if ((offset - 1) % 20 != 0)
            {
                remainder = 20 % ((offset - 1) % 20);
            }

            int upperLimit = lowerLimit + (limit - 1 - remainder) / 20;

            if (offset - 1 + limit > 20 * (upperLimit + 1))
            {
                upperLimit++;
            }

            for (int idx = lowerLimit; idx <= upperLimit; ++idx)
            {
                Urls.Add(string.Format(@"search/{0}/{1}/results?page={2}&sort={3}",
                                       GetThirdLevelRequest(), SearchString.Replace(" ", "%20"), idx, sort));
            }

            return(Urls.Count > 0);
        }
        private void Parse(string html, string previous)
        {
            Uri uri = new Uri(previous);

            foreach (Match match in HREF_REGEX.Matches(html))
            {
                string url = match.Groups["url"].Value;
                if (!URL_REGEX.IsMatch(url))
                {
                    continue;
                }
                Uri    add = new Uri(url, UriKind.RelativeOrAbsolute);
                string result;
                if (url.StartsWith("http://") || url.StartsWith("https://"))
                {
                    if (add.Host != uri.Host)
                    {
                        continue;
                    }
                    result = add.AbsoluteUri;
                }
                else if (!add.IsAbsoluteUri)
                {
                    result = new Uri(uri, add).AbsoluteUri;
                }
                else
                {
                    continue;
                }

                if (!Urls.Contains(result) && !CompletedUrls.Contains(result) && !DownloadingUrls.Contains(result))
                {
                    Urls.Add(result);
                }
            }
        }
Пример #30
0
 public virtual void ReadFrom(XElement xE)
 {
     MediaId      = null;
     Type         = null;
     ReferenceId  = null;
     Dimensions   = null;
     Urls         = null;
     MimeType     = null;
     SourceUrl    = null;
     Name         = null;
     FileSize     = null;
     CreationTime = null;
     MediaType    = null;
     foreach (var xItem in xE.Elements())
     {
         var localName = xItem.Name.LocalName;
         if (localName == "mediaId")
         {
             MediaId = long.Parse(xItem.Value);
         }
         else if (localName == "type")
         {
             Type = MediaMediaTypeExtensions.Parse(xItem.Value);
         }
         else if (localName == "referenceId")
         {
             ReferenceId = long.Parse(xItem.Value);
         }
         else if (localName == "dimensions")
         {
             if (Dimensions == null)
             {
                 Dimensions = new List <Media_Size_DimensionsMapEntry>();
             }
             var dimensionsItem = new Media_Size_DimensionsMapEntry();
             dimensionsItem.ReadFrom(xItem);
             Dimensions.Add(dimensionsItem);
         }
         else if (localName == "urls")
         {
             if (Urls == null)
             {
                 Urls = new List <Media_Size_StringMapEntry>();
             }
             var urlsItem = new Media_Size_StringMapEntry();
             urlsItem.ReadFrom(xItem);
             Urls.Add(urlsItem);
         }
         else if (localName == "mimeType")
         {
             MimeType = MediaMimeTypeExtensions.Parse(xItem.Value);
         }
         else if (localName == "sourceUrl")
         {
             SourceUrl = xItem.Value;
         }
         else if (localName == "name")
         {
             Name = xItem.Value;
         }
         else if (localName == "fileSize")
         {
             FileSize = long.Parse(xItem.Value);
         }
         else if (localName == "creationTime")
         {
             CreationTime = xItem.Value;
         }
         else if (localName == "Media.Type")
         {
             MediaType = xItem.Value;
         }
     }
 }
Пример #31
0
        private void LoadPage(string url, ufFormatDescriber format)
        {
            if (url != string.Empty)
            {
                Url urlReport = new Url();
                urlReport.Address = url;
                //m_oFormatDescriber = format;

                UrlModule urlModule = new UrlModule();
                Uri uri = new Uri(url);
                urlModule.DocumentContentType = UrlModule.ContentType.Html;
                urlModule.DocumentRequestType = UrlModule.RequestType.Get;
                urlModule.Load(uri);

                urlReport.Status = urlModule.StatusCode;
                DateTime started = DateTime.Now;
                m_AllParsedUrls.Add(urlReport);

                if (urlModule.StatusCode == 200 && urlModule.Html != null)
                    ParseUf(urlModule.Html, uri.ToString(), format);

                //----------------------
                Urls newUrls = new Urls();
                m_oMe.Nodes.Add(m_oData);

                foreach(ufDateNode node in m_oData.Nodes)
                {
                    if (node.Nodes["link"] != null)
                    {
                        string link = node.Nodes["link"].Value;
                        if (ShouldParseUrl(link))
                            newUrls.Add(new Url(link));
                    }
                }
                foreach (Url newUrl in newUrls)
                {
                    LoadPage(newUrl.Address, m_oMeDescriber);
                }
                //----------------------

                DateTime ended = DateTime.Now;
                urlReport.LoadTime = ended.Subtract(started);
            }
        }