Exemplo n.º 1
0
 private void Check(UserAgent agent, string name, string expectedValue, long expectedConfidence)
 {
     agent.Get(name).GetValue().Should().Be(expectedValue);
     agent.GetValue(name).Should().Be(expectedValue);
     agent.Get(name).GetConfidence().Should().Be(expectedConfidence);
     agent.GetConfidence(name).Should().Be(expectedConfidence);
 }
        /// <summary>
        /// Calculate the major version from other fields.
        /// </summary>
        /// <param name="userAgent">The <see cref="UserAgent"/>.</param>
        public void Calculate(UserAgent userAgent)
        {
            var agentVersionMajor = userAgent.Get(this.majorVersionName);

            if (agentVersionMajor is null || agentVersionMajor.GetConfidence() == -1)
            {
                var agentVersion = userAgent.Get(this.versionName);
                if (agentVersion != null)
                {
                    string version = agentVersion.GetValue();
                    if (version != null)
                    {
                        version = VersionSplitter.Instance.GetSingleSplit(agentVersion.GetValue(), 1);
                    }
                    else
                    {
                        version = "??";
                    }

                    userAgent.SetForced(
                        this.majorVersionName,
                        version,
                        agentVersion.GetConfidence());
                }
            }
        }
        /// <summary>
        /// Calculate the <see cref="DefaultUserAgentFields.DEVICE_NAME"/> field.
        /// </summary>
        /// <param name="userAgent">The <see cref="UserAgent"/>.</param>
        public void Calculate(UserAgent userAgent)
        {
            // Make sure the DeviceName always starts with the DeviceBrand
            var deviceName = userAgent.Get(DefaultUserAgentFields.DEVICE_NAME);

            if (deviceName.GetConfidence() >= 0)
            {
                var    deviceBrand      = userAgent.Get(DefaultUserAgentFields.DEVICE_BRAND);
                string deviceNameValue  = this.RemoveBadSubStrings(deviceName.GetValue());
                string deviceBrandValue = deviceBrand.GetValue();
                if (deviceName.GetConfidence() >= 0 &&
                    deviceBrand.GetConfidence() >= 0 &&
                    !deviceBrandValue.Equals("Unknown"))
                {
                    // In some cases it does start with the brand but without a separator following the brand
                    deviceNameValue = Normalize.CleanupDeviceBrandName(deviceBrandValue, deviceNameValue);
                }
                else
                {
                    deviceNameValue = Normalize.Brand(deviceNameValue);
                }

                userAgent.SetForced(
                    DefaultUserAgentFields.DEVICE_NAME,
                    deviceNameValue,
                    deviceName.GetConfidence());
            }
        }
Exemplo n.º 4
0
        public void Useragent()
        {
            string    uaString = "Foo Bar";
            UserAgent agent    = new UserAgent(uaString);

            agent.Get(DefaultUserAgentFields.USERAGENT_FIELDNAME).GetValue().Should().Be(uaString);
            agent.Get(DefaultUserAgentFields.USERAGENT_FIELDNAME).GetConfidence().Should().Be(0);
            agent.GetValue(DefaultUserAgentFields.USERAGENT_FIELDNAME).Should().Be(uaString);
            agent.GetConfidence(DefaultUserAgentFields.USERAGENT_FIELDNAME).Should().Be(0L);
        }
 public Specification when_collecting_user_input_for_a_command()
 {
     return(new NancyResponseSpecification <TestCommandModule>
     {
         OnContext = context =>
         {
             context.Header("Accept", "text/html");
             context.Query("name", "Ninja Pro");
             context.Query("description", "Allows you to move silently.");
         },
         When = context => UserAgent.Get("/dispatcher/test", context),
         Expect =
         {
             result => result.Response.StatusCode.Equals(HttpStatusCode.OK),
             result => result.Response.Body.AsString().Contains("Razor Compilation Error").Equals(false),
             result => result.Response.Body["form"].Count().Equals(1),
             result => result.Response.Body["form"].Single().Attributes["method"] == "POST",
             result => result.Response.Body["form"].Single().Attributes["action"] == "/dispatcher/test",
             result => result.Response.Body["form input[name='name']"].Count().Equals(1),
             result => result.Response.Body["form input[name='name']"].Single().Attributes["type"].Equals("text"),
             result => result.Response.Body["form input[name='name']"].Single().Attributes["value"].Equals("Ninja Pro"),
             result => result.Response.Body["form input[name='date']"].Single().Attributes["type"].Equals("datetime"),
             result => result.Response.Body["form input[name='isChecked']"].Single().Attributes["type"].Equals("checkbox"),
         }
     });
 }
        /// <summary>
        /// Calculate the <see cref="DefaultUserAgentFields.DEVICE_BRAND"/> field.
        /// </summary>
        /// <param name="userAgent">The <see cref="UserAgent"/>.</param>
        public void Calculate(UserAgent userAgent)
        {
            // The device brand field is a mess.
            var deviceBrand = userAgent.Get(DefaultUserAgentFields.DEVICE_BRAND);

            if (!deviceBrand.IsDefaultValue)
            {
                userAgent.SetForced(
                    DefaultUserAgentFields.DEVICE_BRAND,
                    Normalize.Brand(deviceBrand.GetValue()),
                    deviceBrand.GetConfidence());
            }

            if (deviceBrand.IsDefaultValue)
            {
                // If no brand is known then try to extract something that looks like a Brand from things like URL and Email addresses.
                var newDeviceBrand = this.DetermineDeviceBrand(userAgent);
                if (newDeviceBrand != null)
                {
                    userAgent.SetForced(
                        DefaultUserAgentFields.DEVICE_BRAND,
                        newDeviceBrand,
                        1);
                }
            }
        }
        /// <summary>
        /// Tries to determine the device brand from other fields (like email and url).
        /// </summary>
        /// <param name="userAgent">The <see cref="UserAgent"/>.</param>
        /// <returns>The device brand.</returns>
        private string DetermineDeviceBrand(UserAgent userAgent)
        {
            // If no brand is known but we do have a URL then we assume the hostname to be the brand.
            // We put this AFTER the creation of the DeviceName because we choose to not have
            // this brandname in the DeviceName.
            var informationUrl = userAgent.Get(DefaultUserAgentFields.AGENT_INFORMATION_URL);

            if (informationUrl != null && informationUrl.GetConfidence() >= 0)
            {
                var hostname = informationUrl.GetValue();
                try
                {
                    var url = new Uri(hostname);
                    hostname = url.Host;
                }
                catch (Exception)
                {
                    // Ignore any exception and continue.
                }

                hostname = this.ExtractCompanyFromHostName(hostname, this.unwantedUrlBrands);
                if (hostname != null)
                {
                    return(hostname);
                }
            }

            var informationEmail = userAgent.Get(DefaultUserAgentFields.AGENT_INFORMATION_EMAIL);

            if (informationEmail != null && informationEmail.GetConfidence() >= 0)
            {
                var hostname = informationEmail.GetValue();
                var atOffset = hostname.IndexOf('@');
                if (atOffset >= 0)
                {
                    hostname = hostname.Substring(atOffset + 1);
                }

                hostname = this.ExtractCompanyFromHostName(hostname, this.unwantedEmailBrands);
                if (hostname != null)
                {
                    return(hostname);
                }
            }

            return(null);
        }
Exemplo n.º 8
0
        /// <summary>
        /// Calculate the <see cref="DefaultUserAgentFields.AGENT_INFORMATION_EMAIL"/> field.
        /// </summary>
        /// <param name="userAgent">The <see cref="UserAgent"/>.</param>
        public void Calculate(UserAgent userAgent)
        {
            // The email address is a mess
            var email = userAgent.Get(DefaultUserAgentFields.AGENT_INFORMATION_EMAIL);

            if (email != null && email.GetConfidence() >= 0)
            {
                userAgent.SetForced(
                    DefaultUserAgentFields.AGENT_INFORMATION_EMAIL,
                    Normalize.Email(email.GetValue()),
                    email.GetConfidence());
            }
        }
Exemplo n.º 9
0
        /// <summary>
        /// Calculate the <see cref="DefaultUserAgentFields.NETWORK_TYPE"/> field.
        /// </summary>
        /// <param name="userAgent">The <see cref="UserAgent"/>.</param>
        public void Calculate(UserAgent userAgent)
        {
            // Make sure the DeviceName always starts with the DeviceBrand
            var networkType = userAgent.Get(DefaultUserAgentFields.NETWORK_TYPE);

            if (networkType != null && networkType.GetConfidence() >= 0)
            {
                userAgent.SetForced(
                    DefaultUserAgentFields.NETWORK_TYPE,
                    Normalize.Brand(networkType.GetValue()),
                    networkType.GetConfidence());
            }
        }
Exemplo n.º 10
0
        static void worker(Object s)
        {
            while (true)
            {
                try
                {
                    using (HttpRequest req = new HttpRequest())
                    {
                        HttpResponse res;
                        Proxy        proxy;
                        string       url;
                        string       urlToGet;
                        string       cl;
                        string       ei;
                        string       of;
                        string       vm;

                        double vol = random.Next(200, 1000) / 10d;

                        proxy = scraper.Next();

                        switch (proxyType)
                        {
                        case Https:
                            req.Proxy = proxy.Http;
                            break;

                        case Public:
                        case Socks4:
                            req.Proxy = proxy.Socks4;
                            break;

                        case Socks5:
                            req.Proxy = proxy.Socks5;
                            break;
                        }

                        req.UserAgent = UserAgent.Get();

                        res = req.Get($"https://m.youtube.com/watch?v={id}");

                        url = url_re.Match(res.ToString()).Groups[1].Value;
                        url = url.Replace(@"\\u0026", "&").Replace("%2C", ",").Replace(@"\/", "/");

                        cl = cl_re.Match(url).Groups[1].Value;
                        ei = ei_re.Match(url).Groups[1].Value;
                        of = of_re.Match(url).Groups[1].Value;
                        vm = vm_re.Match(url).Groups[1].Value;

                        cl = url.Split(new string[] { "cl=" }, StringSplitOptions.None)[1].Split('&')[0];
                        ei = url.Split(new string[] { "ei=" }, StringSplitOptions.None)[1].Split('&')[0];
                        of = url.Split(new string[] { "of=" }, StringSplitOptions.None)[1].Split('&')[0];
                        vm = url.Split(new string[] { "vm=" }, StringSplitOptions.None)[1].Split('&')[0];

                        urlToGet = $"https://s.youtube.com/api/stats/watchtime?ns=yt&el=detailpage&cpn=isWmmj2C9Y2vULKF&docid={id}&ver=2&cmt=7334&ei={ei}&fmt=133&fs=0&rt=1003&of={of}&euri&lact=4418&live=dvr&cl={cl}&state=playing&vm={vm}&volume={vol}&c=MWEB&cver=2.20200313.03.00&cplayer=UNIPLAYER&cbrand=apple&cbr=Safari%20Mobile&cbrver=12.1.15E148&cmodel=iphone&cos=iPhone&cosver=12_2&cplatform=MOBILE&delay=5&hl=ru&cr=GB&rtn=1303&afmt=140&lio=1556394045.182&idpj=&ldpj=&rti=1003&muted=0&st=7334&et=7634";

                        req.AddHeader("Referrer", $"https://m.youtube.com/watch?v={id}");
                        req.AddHeader("Host", "m.youtube.com");
                        req.AddHeader("Proxy-Connection", "keep-alive");
                        req.AddHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
                        req.AddHeader("Accept-Language", "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7");
                        req.AddHeader("Accept-Encoding", "gzip, deflate");

                        req.Get(urlToGet);

                        Interlocked.Increment(ref botted);
                        log();
                    }
                }
                catch
                {
                    Interlocked.Increment(ref errors);
                    log();
                }
            }
        }
Exemplo n.º 11
0
        // Module defining this command


        // Optional custom code for this activity


        /// <summary>
        /// Returns a configured instance of System.Management.Automation.PowerShell, pre-populated with the command to run.
        /// </summary>
        /// <param name="context">The NativeActivityContext for the currently running activity.</param>
        /// <returns>A populated instance of Sytem.Management.Automation.PowerShell</returns>
        /// <remarks>The infrastructure takes responsibility for closing and disposing the PowerShell instance returned.</remarks>
        protected override ActivityImplementationContext GetPowerShell(NativeActivityContext context)
        {
            System.Management.Automation.PowerShell invoker       = global::System.Management.Automation.PowerShell.Create();
            System.Management.Automation.PowerShell targetCommand = invoker.AddCommand(PSCommandName);

            // Initialize the arguments

            if (Method.Expression != null)
            {
                targetCommand.AddParameter("Method", Method.Get(context));
            }

            if (UseBasicParsing.Expression != null)
            {
                targetCommand.AddParameter("UseBasicParsing", UseBasicParsing.Get(context));
            }

            if (Uri.Expression != null)
            {
                targetCommand.AddParameter("Uri", Uri.Get(context));
            }

            if (WebSession.Expression != null)
            {
                targetCommand.AddParameter("WebSession", WebSession.Get(context));
            }

            if (SessionVariable.Expression != null)
            {
                targetCommand.AddParameter("SessionVariable", SessionVariable.Get(context));
            }

            if (Credential.Expression != null)
            {
                targetCommand.AddParameter("Credential", Credential.Get(context));
            }

            if (UseDefaultCredentials.Expression != null)
            {
                targetCommand.AddParameter("UseDefaultCredentials", UseDefaultCredentials.Get(context));
            }

            if (CertificateThumbprint.Expression != null)
            {
                targetCommand.AddParameter("CertificateThumbprint", CertificateThumbprint.Get(context));
            }

            if (Certificate.Expression != null)
            {
                targetCommand.AddParameter("Certificate", Certificate.Get(context));
            }

            if (UserAgent.Expression != null)
            {
                targetCommand.AddParameter("UserAgent", UserAgent.Get(context));
            }

            if (DisableKeepAlive.Expression != null)
            {
                targetCommand.AddParameter("DisableKeepAlive", DisableKeepAlive.Get(context));
            }

            if (TimeoutSec.Expression != null)
            {
                targetCommand.AddParameter("TimeoutSec", TimeoutSec.Get(context));
            }

            if (Headers.Expression != null)
            {
                targetCommand.AddParameter("Headers", Headers.Get(context));
            }

            if (MaximumRedirection.Expression != null)
            {
                targetCommand.AddParameter("MaximumRedirection", MaximumRedirection.Get(context));
            }

            if (Proxy.Expression != null)
            {
                targetCommand.AddParameter("Proxy", Proxy.Get(context));
            }

            if (ProxyCredential.Expression != null)
            {
                targetCommand.AddParameter("ProxyCredential", ProxyCredential.Get(context));
            }

            if (ProxyUseDefaultCredentials.Expression != null)
            {
                targetCommand.AddParameter("ProxyUseDefaultCredentials", ProxyUseDefaultCredentials.Get(context));
            }

            if (Body.Expression != null)
            {
                targetCommand.AddParameter("Body", Body.Get(context));
            }

            if (ContentType.Expression != null)
            {
                targetCommand.AddParameter("ContentType", ContentType.Get(context));
            }

            if (TransferEncoding.Expression != null)
            {
                targetCommand.AddParameter("TransferEncoding", TransferEncoding.Get(context));
            }

            if (InFile.Expression != null)
            {
                targetCommand.AddParameter("InFile", InFile.Get(context));
            }

            if (OutFile.Expression != null)
            {
                targetCommand.AddParameter("OutFile", OutFile.Get(context));
            }

            if (PassThru.Expression != null)
            {
                targetCommand.AddParameter("PassThru", PassThru.Get(context));
            }


            return(new ActivityImplementationContext()
            {
                PowerShellInstance = invoker
            });
        }
Exemplo n.º 12
0
        private void TestUseragentValuesDebug(bool debug)
        {
            string    name     = "Attribute";
            string    uaString = "Foo Bar";
            UserAgent agent    = new UserAgent(uaString)
            {
                IsDebug = debug
            };

            // Setting unknown new attributes
            agent.Get("UnknownOne").Should().BeNull();
            agent.GetValue("UnknownOne").Should().Be("Unknown");
            agent.GetConfidence("UnknownOne").Should().Be(-1);
            agent.Set("UnknownOne", "One", 111);
            Check(agent, "UnknownOne", "One", 111);
            agent.Get("UnknownOne").Reset();
            Check(agent, "UnknownOne", null, -1);

            // Setting unknown new attributes FORCED
            agent.Get("UnknownTwo").Should().BeNull();
            agent.GetValue("UnknownTwo").Should().Be("Unknown");
            agent.GetConfidence("UnknownTwo").Should().Be(-1);
            agent.SetForced("UnknownTwo", "Two", 222);
            Check(agent, "UnknownTwo", "Two", 222);
            agent.Get("UnknownTwo").Reset();
            Check(agent, "UnknownTwo", null, -1);

            // Setting known attributes
            Check(agent, "AgentClass", "Unknown", -1);
            agent.Set("AgentClass", "One", 111);
            Check(agent, "AgentClass", "One", 111);
            agent.Get("AgentClass").Reset();
            Check(agent, "AgentClass", "Unknown", -1);

            // Setting known attributes FORCED
            Check(agent, "AgentVersion", "??", -1);
            agent.SetForced("AgentVersion", "Two", 222);
            Check(agent, "AgentVersion", "Two", 222);
            agent.Get("AgentVersion").Reset();
            Check(agent, "AgentVersion", "??", -1);

            agent.Set(name, "One", 111);
            Check(agent, name, "One", 111);

            agent.Set(name, "Two", 22); // Should be ignored
            Check(agent, name, "One", 111);

            agent.Set(name, "Three", 333); // Should be used
            Check(agent, name, "Three", 333);

            agent.SetForced(name, "Four", 4); // Should be used
            Check(agent, name, "Four", 4);

            agent.Set(name, "<<<null>>>", 2); // Should be ignored
            Check(agent, name, "Four", 4);

            agent.Set(name, "<<<null>>>", 5); // Should be used
            Check(agent, name, null, -1);     // -1 --> SPECIAL CASE!!!

            agent.Set(name, "Four", 4);       // Should be IGNORED (special case remember)
            Check(agent, name, null, -1);     // -1 --> SPECIAL CASE!!!

            // Set a 'normal' value again.
            agent.Set(name, "Three", 333); // Should be used
            Check(agent, name, "Three", 333);

            var field = agent.Get(name);

            field.SetValueForced("Five", 5);       // Should be used
            Check(agent, name, "Five", 5);
            field.SetValueForced("<<<null>>>", 4); // Should be used
            Check(agent, name, null, -1);          // -1 --> SPECIAL CASE!!!
        }
        /// <summary>
        /// Calculate a derived field from a concatenation of other fields.
        /// </summary>
        /// <param name="userAgent">The <see cref="UserAgent"/>.</param>
        public void Calculate(UserAgent userAgent)
        {
            var firstField  = userAgent.Get(this.firstName);
            var secondField = userAgent.Get(this.secondName);

            string first            = null;
            long   firstConfidence  = -1;
            string second           = null;
            long   secondConfidence = -1;

            if (firstField != null)
            {
                first           = firstField.GetValue();
                firstConfidence = firstField.GetConfidence();
            }

            if (secondField != null)
            {
                second           = secondField.GetValue();
                secondConfidence = secondField.GetConfidence();
            }

            if (first is null && second is null)
            {
                return; // Nothing to do
            }

            if (second is null)
            {
                if (firstConfidence >= 0)
                {
                    userAgent.Set(this.targetName, first, firstConfidence);
                }
                else
                {
                    userAgent.SetForced(this.targetName, "Unknown", firstConfidence);
                }

                return; // Nothing to do
            }
            else
            {
                if (first is null)
                {
                    if (secondConfidence >= 0)
                    {
                        userAgent.Set(this.targetName, second, secondConfidence);
                    }
                    else
                    {
                        userAgent.SetForced(this.targetName, "Unknown", secondConfidence);
                    }

                    return;
                }
            }

            if (first.Equals(second))
            {
                userAgent.Set(this.targetName, first, firstConfidence);
            }
            else
            {
                if (second.StartsWith(first))
                {
                    userAgent.Set(this.targetName, second, secondConfidence);
                }
                else
                {
                    var  value      = first + " " + second;
                    long confidence = Math.Max(firstConfidence, secondConfidence);
                    if (confidence < 0)
                    {
                        userAgent.SetForced(this.targetName, value, confidence);
                    }
                    else
                    {
                        userAgent.Set(this.targetName, value, confidence);
                    }
                }
            }
        }