private void Check(UserAgent agent, string name, string expectedValue, long expectedConfidence)
{
agent.Get(name).GetValue().Should().Be(expectedValue);
agent.GetValue(name).Should().Be(expectedValue);
agent.Get(name).GetConfidence().Should().Be(expectedConfidence);
agent.GetConfidence(name).Should().Be(expectedConfidence);
}
/// <summary>
/// Calculate the major version from other fields.
/// </summary>
/// <param name="userAgent">The <see cref="UserAgent"/>.</param>
public void Calculate(UserAgent userAgent)
{
var agentVersionMajor = userAgent.Get(this.majorVersionName);
if (agentVersionMajor is null || agentVersionMajor.GetConfidence() == -1)
{
var agentVersion = userAgent.Get(this.versionName);
if (agentVersion != null)
{
string version = agentVersion.GetValue();
if (version != null)
{
version = VersionSplitter.Instance.GetSingleSplit(agentVersion.GetValue(), 1);
}
else
{
version = "??";
}
userAgent.SetForced(
this.majorVersionName,
version,
agentVersion.GetConfidence());
}
}
}
/// <summary>
/// Calculate the <see cref="DefaultUserAgentFields.DEVICE_NAME"/> field.
/// </summary>
/// <param name="userAgent">The <see cref="UserAgent"/>.</param>
public void Calculate(UserAgent userAgent)
{
// Make sure the DeviceName always starts with the DeviceBrand
var deviceName = userAgent.Get(DefaultUserAgentFields.DEVICE_NAME);
if (deviceName.GetConfidence() >= 0)
{
var deviceBrand = userAgent.Get(DefaultUserAgentFields.DEVICE_BRAND);
string deviceNameValue = this.RemoveBadSubStrings(deviceName.GetValue());
string deviceBrandValue = deviceBrand.GetValue();
if (deviceName.GetConfidence() >= 0 &&
deviceBrand.GetConfidence() >= 0 &&
!deviceBrandValue.Equals("Unknown"))
{
// In some cases it does start with the brand but without a separator following the brand
deviceNameValue = Normalize.CleanupDeviceBrandName(deviceBrandValue, deviceNameValue);
}
else
{
deviceNameValue = Normalize.Brand(deviceNameValue);
}
userAgent.SetForced(
DefaultUserAgentFields.DEVICE_NAME,
deviceNameValue,
deviceName.GetConfidence());
}
}
public void Useragent()
{
string uaString = "Foo Bar";
UserAgent agent = new UserAgent(uaString);
agent.Get(DefaultUserAgentFields.USERAGENT_FIELDNAME).GetValue().Should().Be(uaString);
agent.Get(DefaultUserAgentFields.USERAGENT_FIELDNAME).GetConfidence().Should().Be(0);
agent.GetValue(DefaultUserAgentFields.USERAGENT_FIELDNAME).Should().Be(uaString);
agent.GetConfidence(DefaultUserAgentFields.USERAGENT_FIELDNAME).Should().Be(0L);
}
public Specification when_collecting_user_input_for_a_command()
{
return(new NancyResponseSpecification <TestCommandModule>
{
OnContext = context =>
{
context.Header("Accept", "text/html");
context.Query("name", "Ninja Pro");
context.Query("description", "Allows you to move silently.");
},
When = context => UserAgent.Get("/dispatcher/test", context),
Expect =
{
result => result.Response.StatusCode.Equals(HttpStatusCode.OK),
result => result.Response.Body.AsString().Contains("Razor Compilation Error").Equals(false),
result => result.Response.Body["form"].Count().Equals(1),
result => result.Response.Body["form"].Single().Attributes["method"] == "POST",
result => result.Response.Body["form"].Single().Attributes["action"] == "/dispatcher/test",
result => result.Response.Body["form input[name='name']"].Count().Equals(1),
result => result.Response.Body["form input[name='name']"].Single().Attributes["type"].Equals("text"),
result => result.Response.Body["form input[name='name']"].Single().Attributes["value"].Equals("Ninja Pro"),
result => result.Response.Body["form input[name='date']"].Single().Attributes["type"].Equals("datetime"),
result => result.Response.Body["form input[name='isChecked']"].Single().Attributes["type"].Equals("checkbox"),
}
});
}
/// <summary>
/// Calculate the <see cref="DefaultUserAgentFields.DEVICE_BRAND"/> field.
/// </summary>
/// <param name="userAgent">The <see cref="UserAgent"/>.</param>
public void Calculate(UserAgent userAgent)
{
// The device brand field is a mess.
var deviceBrand = userAgent.Get(DefaultUserAgentFields.DEVICE_BRAND);
if (!deviceBrand.IsDefaultValue)
{
userAgent.SetForced(
DefaultUserAgentFields.DEVICE_BRAND,
Normalize.Brand(deviceBrand.GetValue()),
deviceBrand.GetConfidence());
}
if (deviceBrand.IsDefaultValue)
{
// If no brand is known then try to extract something that looks like a Brand from things like URL and Email addresses.
var newDeviceBrand = this.DetermineDeviceBrand(userAgent);
if (newDeviceBrand != null)
{
userAgent.SetForced(
DefaultUserAgentFields.DEVICE_BRAND,
newDeviceBrand,
1);
}
}
}
/// <summary>
/// Tries to determine the device brand from other fields (like email and url).
/// </summary>
/// <param name="userAgent">The <see cref="UserAgent"/>.</param>
/// <returns>The device brand.</returns>
private string DetermineDeviceBrand(UserAgent userAgent)
{
// If no brand is known but we do have a URL then we assume the hostname to be the brand.
// We put this AFTER the creation of the DeviceName because we choose to not have
// this brandname in the DeviceName.
var informationUrl = userAgent.Get(DefaultUserAgentFields.AGENT_INFORMATION_URL);
if (informationUrl != null && informationUrl.GetConfidence() >= 0)
{
var hostname = informationUrl.GetValue();
try
{
var url = new Uri(hostname);
hostname = url.Host;
}
catch (Exception)
{
// Ignore any exception and continue.
}
hostname = this.ExtractCompanyFromHostName(hostname, this.unwantedUrlBrands);
if (hostname != null)
{
return(hostname);
}
}
var informationEmail = userAgent.Get(DefaultUserAgentFields.AGENT_INFORMATION_EMAIL);
if (informationEmail != null && informationEmail.GetConfidence() >= 0)
{
var hostname = informationEmail.GetValue();
var atOffset = hostname.IndexOf('@');
if (atOffset >= 0)
{
hostname = hostname.Substring(atOffset + 1);
}
hostname = this.ExtractCompanyFromHostName(hostname, this.unwantedEmailBrands);
if (hostname != null)
{
return(hostname);
}
}
return(null);
}
/// <summary>
/// Calculate the <see cref="DefaultUserAgentFields.AGENT_INFORMATION_EMAIL"/> field.
/// </summary>
/// <param name="userAgent">The <see cref="UserAgent"/>.</param>
public void Calculate(UserAgent userAgent)
{
// The email address is a mess
var email = userAgent.Get(DefaultUserAgentFields.AGENT_INFORMATION_EMAIL);
if (email != null && email.GetConfidence() >= 0)
{
userAgent.SetForced(
DefaultUserAgentFields.AGENT_INFORMATION_EMAIL,
Normalize.Email(email.GetValue()),
email.GetConfidence());
}
}
/// <summary>
/// Calculate the <see cref="DefaultUserAgentFields.NETWORK_TYPE"/> field.
/// </summary>
/// <param name="userAgent">The <see cref="UserAgent"/>.</param>
public void Calculate(UserAgent userAgent)
{
// Make sure the DeviceName always starts with the DeviceBrand
var networkType = userAgent.Get(DefaultUserAgentFields.NETWORK_TYPE);
if (networkType != null && networkType.GetConfidence() >= 0)
{
userAgent.SetForced(
DefaultUserAgentFields.NETWORK_TYPE,
Normalize.Brand(networkType.GetValue()),
networkType.GetConfidence());
}
}
static void worker(Object s)
{
while (true)
{
try
{
using (HttpRequest req = new HttpRequest())
{
HttpResponse res;
Proxy proxy;
string url;
string urlToGet;
string cl;
string ei;
string of;
string vm;
double vol = random.Next(200, 1000) / 10d;
proxy = scraper.Next();
switch (proxyType)
{
case Https:
req.Proxy = proxy.Http;
break;
case Public:
case Socks4:
req.Proxy = proxy.Socks4;
break;
case Socks5:
req.Proxy = proxy.Socks5;
break;
}
req.UserAgent = UserAgent.Get();
res = req.Get($"https://m.youtube.com/watch?v={id}");
url = url_re.Match(res.ToString()).Groups[1].Value;
url = url.Replace(@"\\u0026", "&").Replace("%2C", ",").Replace(@"\/", "/");
cl = cl_re.Match(url).Groups[1].Value;
ei = ei_re.Match(url).Groups[1].Value;
of = of_re.Match(url).Groups[1].Value;
vm = vm_re.Match(url).Groups[1].Value;
cl = url.Split(new string[] { "cl=" }, StringSplitOptions.None)[1].Split('&')[0];
ei = url.Split(new string[] { "ei=" }, StringSplitOptions.None)[1].Split('&')[0];
of = url.Split(new string[] { "of=" }, StringSplitOptions.None)[1].Split('&')[0];
vm = url.Split(new string[] { "vm=" }, StringSplitOptions.None)[1].Split('&')[0];
urlToGet = $"https://s.youtube.com/api/stats/watchtime?ns=yt&el=detailpage&cpn=isWmmj2C9Y2vULKF&docid={id}&ver=2&cmt=7334&ei={ei}&fmt=133&fs=0&rt=1003&of={of}&euri&lact=4418&live=dvr&cl={cl}&state=playing&vm={vm}&volume={vol}&c=MWEB&cver=2.20200313.03.00&cplayer=UNIPLAYER&cbrand=apple&cbr=Safari%20Mobile&cbrver=12.1.15E148&cmodel=iphone&cos=iPhone&cosver=12_2&cplatform=MOBILE&delay=5&hl=ru&cr=GB&rtn=1303&afmt=140&lio=1556394045.182&idpj=&ldpj=&rti=1003&muted=0&st=7334&et=7634";
req.AddHeader("Referrer", $"https://m.youtube.com/watch?v={id}");
req.AddHeader("Host", "m.youtube.com");
req.AddHeader("Proxy-Connection", "keep-alive");
req.AddHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
req.AddHeader("Accept-Language", "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7");
req.AddHeader("Accept-Encoding", "gzip, deflate");
req.Get(urlToGet);
Interlocked.Increment(ref botted);
log();
}
}
catch
{
Interlocked.Increment(ref errors);
log();
}
}
}
// Module defining this command
// Optional custom code for this activity
/// <summary>
/// Returns a configured instance of System.Management.Automation.PowerShell, pre-populated with the command to run.
/// </summary>
/// <param name="context">The NativeActivityContext for the currently running activity.</param>
/// <returns>A populated instance of Sytem.Management.Automation.PowerShell</returns>
/// <remarks>The infrastructure takes responsibility for closing and disposing the PowerShell instance returned.</remarks>
protected override ActivityImplementationContext GetPowerShell(NativeActivityContext context)
{
System.Management.Automation.PowerShell invoker = global::System.Management.Automation.PowerShell.Create();
System.Management.Automation.PowerShell targetCommand = invoker.AddCommand(PSCommandName);
// Initialize the arguments
if (Method.Expression != null)
{
targetCommand.AddParameter("Method", Method.Get(context));
}
if (UseBasicParsing.Expression != null)
{
targetCommand.AddParameter("UseBasicParsing", UseBasicParsing.Get(context));
}
if (Uri.Expression != null)
{
targetCommand.AddParameter("Uri", Uri.Get(context));
}
if (WebSession.Expression != null)
{
targetCommand.AddParameter("WebSession", WebSession.Get(context));
}
if (SessionVariable.Expression != null)
{
targetCommand.AddParameter("SessionVariable", SessionVariable.Get(context));
}
if (Credential.Expression != null)
{
targetCommand.AddParameter("Credential", Credential.Get(context));
}
if (UseDefaultCredentials.Expression != null)
{
targetCommand.AddParameter("UseDefaultCredentials", UseDefaultCredentials.Get(context));
}
if (CertificateThumbprint.Expression != null)
{
targetCommand.AddParameter("CertificateThumbprint", CertificateThumbprint.Get(context));
}
if (Certificate.Expression != null)
{
targetCommand.AddParameter("Certificate", Certificate.Get(context));
}
if (UserAgent.Expression != null)
{
targetCommand.AddParameter("UserAgent", UserAgent.Get(context));
}
if (DisableKeepAlive.Expression != null)
{
targetCommand.AddParameter("DisableKeepAlive", DisableKeepAlive.Get(context));
}
if (TimeoutSec.Expression != null)
{
targetCommand.AddParameter("TimeoutSec", TimeoutSec.Get(context));
}
if (Headers.Expression != null)
{
targetCommand.AddParameter("Headers", Headers.Get(context));
}
if (MaximumRedirection.Expression != null)
{
targetCommand.AddParameter("MaximumRedirection", MaximumRedirection.Get(context));
}
if (Proxy.Expression != null)
{
targetCommand.AddParameter("Proxy", Proxy.Get(context));
}
if (ProxyCredential.Expression != null)
{
targetCommand.AddParameter("ProxyCredential", ProxyCredential.Get(context));
}
if (ProxyUseDefaultCredentials.Expression != null)
{
targetCommand.AddParameter("ProxyUseDefaultCredentials", ProxyUseDefaultCredentials.Get(context));
}
if (Body.Expression != null)
{
targetCommand.AddParameter("Body", Body.Get(context));
}
if (ContentType.Expression != null)
{
targetCommand.AddParameter("ContentType", ContentType.Get(context));
}
if (TransferEncoding.Expression != null)
{
targetCommand.AddParameter("TransferEncoding", TransferEncoding.Get(context));
}
if (InFile.Expression != null)
{
targetCommand.AddParameter("InFile", InFile.Get(context));
}
if (OutFile.Expression != null)
{
targetCommand.AddParameter("OutFile", OutFile.Get(context));
}
if (PassThru.Expression != null)
{
targetCommand.AddParameter("PassThru", PassThru.Get(context));
}
return(new ActivityImplementationContext()
{
PowerShellInstance = invoker
});
}
private void TestUseragentValuesDebug(bool debug)
{
string name = "Attribute";
string uaString = "Foo Bar";
UserAgent agent = new UserAgent(uaString)
{
IsDebug = debug
};
// Setting unknown new attributes
agent.Get("UnknownOne").Should().BeNull();
agent.GetValue("UnknownOne").Should().Be("Unknown");
agent.GetConfidence("UnknownOne").Should().Be(-1);
agent.Set("UnknownOne", "One", 111);
Check(agent, "UnknownOne", "One", 111);
agent.Get("UnknownOne").Reset();
Check(agent, "UnknownOne", null, -1);
// Setting unknown new attributes FORCED
agent.Get("UnknownTwo").Should().BeNull();
agent.GetValue("UnknownTwo").Should().Be("Unknown");
agent.GetConfidence("UnknownTwo").Should().Be(-1);
agent.SetForced("UnknownTwo", "Two", 222);
Check(agent, "UnknownTwo", "Two", 222);
agent.Get("UnknownTwo").Reset();
Check(agent, "UnknownTwo", null, -1);
// Setting known attributes
Check(agent, "AgentClass", "Unknown", -1);
agent.Set("AgentClass", "One", 111);
Check(agent, "AgentClass", "One", 111);
agent.Get("AgentClass").Reset();
Check(agent, "AgentClass", "Unknown", -1);
// Setting known attributes FORCED
Check(agent, "AgentVersion", "??", -1);
agent.SetForced("AgentVersion", "Two", 222);
Check(agent, "AgentVersion", "Two", 222);
agent.Get("AgentVersion").Reset();
Check(agent, "AgentVersion", "??", -1);
agent.Set(name, "One", 111);
Check(agent, name, "One", 111);
agent.Set(name, "Two", 22); // Should be ignored
Check(agent, name, "One", 111);
agent.Set(name, "Three", 333); // Should be used
Check(agent, name, "Three", 333);
agent.SetForced(name, "Four", 4); // Should be used
Check(agent, name, "Four", 4);
agent.Set(name, "<<<null>>>", 2); // Should be ignored
Check(agent, name, "Four", 4);
agent.Set(name, "<<<null>>>", 5); // Should be used
Check(agent, name, null, -1); // -1 --> SPECIAL CASE!!!
agent.Set(name, "Four", 4); // Should be IGNORED (special case remember)
Check(agent, name, null, -1); // -1 --> SPECIAL CASE!!!
// Set a 'normal' value again.
agent.Set(name, "Three", 333); // Should be used
Check(agent, name, "Three", 333);
var field = agent.Get(name);
field.SetValueForced("Five", 5); // Should be used
Check(agent, name, "Five", 5);
field.SetValueForced("<<<null>>>", 4); // Should be used
Check(agent, name, null, -1); // -1 --> SPECIAL CASE!!!
}
/// <summary>
/// Calculate a derived field from a concatenation of other fields.
/// </summary>
/// <param name="userAgent">The <see cref="UserAgent"/>.</param>
public void Calculate(UserAgent userAgent)
{
var firstField = userAgent.Get(this.firstName);
var secondField = userAgent.Get(this.secondName);
string first = null;
long firstConfidence = -1;
string second = null;
long secondConfidence = -1;
if (firstField != null)
{
first = firstField.GetValue();
firstConfidence = firstField.GetConfidence();
}
if (secondField != null)
{
second = secondField.GetValue();
secondConfidence = secondField.GetConfidence();
}
if (first is null && second is null)
{
return; // Nothing to do
}
if (second is null)
{
if (firstConfidence >= 0)
{
userAgent.Set(this.targetName, first, firstConfidence);
}
else
{
userAgent.SetForced(this.targetName, "Unknown", firstConfidence);
}
return; // Nothing to do
}
else
{
if (first is null)
{
if (secondConfidence >= 0)
{
userAgent.Set(this.targetName, second, secondConfidence);
}
else
{
userAgent.SetForced(this.targetName, "Unknown", secondConfidence);
}
return;
}
}
if (first.Equals(second))
{
userAgent.Set(this.targetName, first, firstConfidence);
}
else
{
if (second.StartsWith(first))
{
userAgent.Set(this.targetName, second, secondConfidence);
}
else
{
var value = first + " " + second;
long confidence = Math.Max(firstConfidence, secondConfidence);
if (confidence < 0)
{
userAgent.SetForced(this.targetName, value, confidence);
}
else
{
userAgent.Set(this.targetName, value, confidence);
}
}
}
}
