public void TestCacheSetter()
{
var userAgentAnalyzer = UserAgentAnalyzer.NewBuilder().Build();
userAgentAnalyzer.LoadResources(Config.RESOURCES_PATH, "*-tests.yaml");
userAgentAnalyzer.CacheSize.Should().Be(10000, "Default cache size");
userAgentAnalyzer.SetCacheSize(50);
userAgentAnalyzer.CacheSize.Should().Be(50, "I set that size");
userAgentAnalyzer.SetCacheSize(50000);
userAgentAnalyzer.CacheSize.Should().Be(50000, "I set that size");
userAgentAnalyzer.SetCacheSize(-5);
userAgentAnalyzer.CacheSize.Should().Be(0, "I set incorrect cache size");
userAgentAnalyzer.SetCacheSize(50);
userAgentAnalyzer.CacheSize.Should().Be(50, "I set that size");
userAgentAnalyzer.SetCacheSize(50000);
userAgentAnalyzer.CacheSize.Should().Be(50000, "I set that size");
userAgentAnalyzer.SetUserAgentMaxLength(555);
userAgentAnalyzer.GetUserAgentMaxLength().Should().Be(555, "I set that size");
}
public void ProfileMemoryFootprint()
{
Skip.If(!Config.ENABLE_PROFILING);
PrintCurrentMemoryProfile("Before ");
UserAgentAnalyzer uaa = UserAgentAnalyzer
.NewBuilder()
.HideMatcherLoadStats()
.WithoutCache()
.KeepTests()
.Build();
PrintCurrentMemoryProfile("Loaded ");
uaa.InitializeMatchers();
PrintCurrentMemoryProfile("Init ");
GC.Collect();
PrintCurrentMemoryProfile("Post GC");
uaa.SetCacheSize(1000);
uaa.PreHeat();
GC.Collect();
PrintCurrentMemoryProfile("Cache 1K");
uaa.SetCacheSize(10000);
uaa.PreHeat();
GC.Collect();
PrintCurrentMemoryProfile("Cache 10K");
uaa.DropTests();
GC.Collect();
PrintCurrentMemoryProfile("NoTest ");
}
public void CheckForMemoryLeaks()
{
Skip.If(!Config.ENABLE_PROFILING);
UserAgentAnalyzer uaa = UserAgentAnalyzer
.NewBuilder()
.WithoutCache()
// .withField("OperatingSystemName")
// .withField("OperatingSystemVersion")
// .withField("DeviceClass")
.HideMatcherLoadStats()
.KeepTests()
.Build();
LOG.Info("Init complete");
int iterationsDone = 0;
int iterationsPerLoop = 1000;
for (int i = 0; i < 100; i++)
{
Stopwatch stopwatch = Stopwatch.StartNew();
uaa.PreHeat(iterationsPerLoop, false);
stopwatch.Stop();
iterationsDone += iterationsPerLoop;
long averageNanos = (stopwatch.ElapsedMilliseconds) / iterationsPerLoop;
PrintMemoryUsage(iterationsDone, averageNanos);
}
}
/// <summary>
/// Initialize the analyzer.
/// </summary>
/// <param name="theMapper">The theMapper<see cref="IUserAgentAnnotationMapper{T}"/>.</param>
public void Initialize(IUserAgentAnnotationMapper <T> theMapper)
{
this.mapper = theMapper;
if (this.mapper is null)
{
throw new InvalidParserConfigurationException("[Initialize] The mapper instance is null.");
}
var classOfTArray = typeof(IUserAgentAnnotationMapper <T>).GenericTypeArguments;
if (classOfTArray is null)
{
throw new InvalidParserConfigurationException("Couldn't find the used generic type of the UserAgentAnnotationMapper.");
}
var classOfT = classOfTArray[0];
this.MapMethods(theMapper, classOfT);
if (this.fieldSetters.Count == 0)
{
throw new InvalidParserConfigurationException("You MUST specify at least 1 field to extract.");
}
this.userAgentAnalyzer = UserAgentAnalyzer
.NewBuilder()
.HideMatcherLoadStats()
.WithCache(this.CacheSize)
.WithFields(this.fieldSetters.Keys)
.DropTests()
.ImmediateInitialization()
.Build();
}
public void TestPostPreheatDroptests()
{
var userAgentAnalyzer =
UserAgentAnalyzer
.NewBuilder()
.ImmediateInitialization()
// Without .preheat(100)
.DropTests()
.HideMatcherLoadStats()
.WithField("AgentName")
.Build();
userAgentAnalyzer.NumberOfTestCases.Should().Be(0);
userAgentAnalyzer =
UserAgentAnalyzer
.NewBuilder()
.ImmediateInitialization()
.Preheat(100) // With .preheat(100)
.DropTests()
.HideMatcherLoadStats()
.WithField("AgentName")
.Build();
userAgentAnalyzer.NumberOfTestCases.Should().Be(0);
}
public void TestCache()
{
var uuid = "11111111-2222-3333-4444-555555555555";
var fieldName = "AgentUuid";
var uaa = UserAgentAnalyzer
.NewBuilder()
.WithCache(1)
.HideMatcherLoadStats()
.WithField(fieldName)
.Build();
UserAgent agent;
uaa.CacheSize.Should().Be(1);
this.GetAllocatedCacheSize(uaa).Should().BeGreaterOrEqualTo(1);
agent = uaa.Parse(uuid);
agent.Get(fieldName).GetValue().Should().Be(uuid);
this.GetCache(uaa)[uuid].Should().BeEquivalentTo(agent);
agent = uaa.Parse(uuid);
agent.Get(fieldName).GetValue().Should().Be(uuid);
this.GetCache(uaa)[uuid].Should().BeEquivalentTo(agent);
uaa.DisableCaching();
uaa.CacheSize.Should().Be(0);
this.GetAllocatedCacheSize(uaa).Should().Be(0);
agent = uaa.Parse(uuid);
agent.Get(fieldName).GetValue().Should().Be(uuid);
this.GetCache(uaa).Should().BeNull();
}
public void GlobalSetup()
{
uaa = UserAgentAnalyzer.NewBuilder()
.WithoutCache()
.HideMatcherLoadStats()
.Build();
uaa.Parse((string)null);
}
static YauaaSingleton()
{
Builder = UserAgentAnalyzer.NewBuilder();
Builder.DropTests();
Builder.DelayInitialization();
Builder.WithCache(100);
Builder.HideMatcherLoadStats();
Builder.WithAllFields();
}
public void TestDualBuilderUsageNoSecondInstance()
{
var builder = UserAgentAnalyzer.NewBuilder().DelayInitialization();
builder.Build().Should().NotBeNull("We should get a first instance from a single builder.");
// And calling build() again should fail with an exception
var a = new Action(() => builder.Build());
a.Should().Throw <Exception>();
}
public void TestDifferentVersion()
{
var uaaB = UserAgentAnalyzer
.NewBuilder()
.DelayInitialization()
.AddResources($"{Config.RESOURCES_PATH}{Path.DirectorySeparatorChar}Versions", "DifferentVersion.yaml");
var a = new Action(() => uaaB.Build());
a.Should().Throw <InvalidParserConfigurationException>().Where(e => e.Message.Contains("Two different Yauaa versions have been loaded:"));
}
public void TestDualBuilderUsageUseSetterAfterBuild()
{
var builder = UserAgentAnalyzer.NewBuilder().DelayInitialization();
builder.Build().Should().NotBeNull("We should get a first instance from a single builder.");
// And calling a setter after the build() should fail with an exception
var a = new Action(() => builder.WithCache(1234));
a.Should().Throw <Exception>();
}
public void TestDoubleLoadedResources()
{
var uaaB = UserAgentAnalyzer
.NewBuilder()
.DelayInitialization()
.AddResources("YamlResources/Useragents");
var a = new Action(() => uaaB.Build());
a.Should().Throw <InvalidParserConfigurationException>().Where(e => e.Message.Contains("resources for the second time"));
}
public void TestBadVersionNotMap()
{
var uaaB = UserAgentAnalyzer
.NewBuilder()
.DropDefaultResources()
.AddResources($"{Config.RESOURCES_PATH}{Path.DirectorySeparatorChar}Versions", "BadVersionNotMap.yaml")
.DelayInitialization();
var a = new Action(() => uaaB.Build());
a.Should().Throw <InvalidParserConfigurationException>().Where(e => e.Message.Contains("The value should be a string but it is a Sequence"));
}
public void TestBadVersion()
{
var uaaB = UserAgentAnalyzer
.NewBuilder()
.DropDefaultResources()
.AddResources($"{Config.RESOURCES_PATH}{Path.DirectorySeparatorChar}Versions", "BadVersion.yaml")
.DelayInitialization();
var a = new Action(() => uaaB.Build());
a.Should().Throw <InvalidParserConfigurationException>().Where(e => e.Message.Contains("Found unexpected config entry: bad"));
}
public void AssesMemoryImpactPerFieldName()
{
Skip.If(!Config.ENABLE_PROFILING);
// Calculate the used memory
var currentProcess = Process.GetCurrentProcess();
long memory = currentProcess.VirtualMemorySize64;
LOG.Error(string.Format(
"Without Yauaa present and GC --> Used memory is %10d bytes (%5d MiB)",
memory, BytesToMegabytes(memory)));
var uaa = UserAgentAnalyzer
.NewBuilder()
.HideMatcherLoadStats()
.WithoutCache()
.KeepTests()
.Build();
uaa.PreHeat();
GC.Collect();
uaa.PreHeat();
GC.Collect();
// Calculate the used memory
memory = currentProcess.VirtualMemorySize64;
LOG.Error(string.Format(
"Querying for 'All fields' and GC --> Used memory is {0} bytes ({1} MiB)",
memory, BytesToMegabytes(memory)));
foreach (string fieldName in uaa.GetAllPossibleFieldNamesSorted())
{
uaa = UserAgentAnalyzer
.NewBuilder()
.WithoutCache()
.WithField(fieldName)
.HideMatcherLoadStats()
.KeepTests()
.Build();
uaa.PreHeat();
GC.Collect();
uaa.PreHeat();
GC.Collect();
// Calculate the used memory
memory = memory = currentProcess.WorkingSet64;
LOG.Error(string.Format(
"Querying for {0} and GC --> Used memory is {1} bytes ({2} MiB)",
fieldName, memory, BytesToMegabytes(memory)));
}
}
public YauaaUserAgentParsingService()
{
_analyzer =
UserAgentAnalyzer
.NewBuilder()
.HideMatcherLoadStats()
.WithField(OrbintSoft.Yauaa.Analyzer.UserAgent.AGENT_NAME)
.WithField(OrbintSoft.Yauaa.Analyzer.UserAgent.AGENT_VERSION)
.WithField(OrbintSoft.Yauaa.Analyzer.UserAgent.DEVICE_CLASS)
.WithField(OrbintSoft.Yauaa.Analyzer.UserAgent.DEVICE_NAME)
.WithCache(10000)
.Build();
}
public void TestLoadMoreResources()
{
var builder = UserAgentAnalyzer.NewBuilder().DelayInitialization().WithField("DeviceClass");
var uaa = builder.Build();
builder.Should().NotBeNull("We should get a first instance from a single builder.");
uaa.InitializeMatchers();
var a = new Action(() => uaa.LoadResources("Something extra"));
a.Should().Throw <Exception>();
}
/// <summary>
/// This method is an helper that does a little trick for us, it reads the actual allocated size in the .NET dictionary using reflection.
/// This method is unsafe, it is used to test the cache mechanism ported by Java, we need ti study a better mechanism for .NET.
/// </summary>
/// <param name="uaa">The <see cref="UserAgentAnalyzer"/> from wich we want read the allocated cache.</param>
/// <returns>The allocated number of elements in the .NET Dictionary</returns>
private int GetAllocatedCacheSize(UserAgentAnalyzer uaa)
{
var cache = this.GetCache(uaa);
if (cache == null)
{
return(0);
}
var bucketsProperty = cache.GetType().GetField("_buckets", BindingFlags.Instance | BindingFlags.NonPublic);
var buckets = bucketsProperty.GetValue(cache) as int[];
return(buckets.Length);
}
public void TestAskingForImpossibleField()
{
var uaa = UserAgentAnalyzer
.NewBuilder()
.WithoutCache()
.HideMatcherLoadStats()
.DelayInitialization()
.WithField("FirstNonexistentField")
.WithField("DeviceClass")
.WithField("SecondNonexistentField");
var a = new Action(() => uaa.Build());
a.Should().Throw <InvalidParserConfigurationException>().WithMessage("We cannot provide these fields: [FirstNonexistentField] [SecondNonexistentField]");
}
/// <summary>
/// The Data
/// </summary>
/// <returns>The <see cref="IEnumerable{object[]}"/></returns>
public static IEnumerable <object[]> Data()
{
var fieldNames = UserAgentAnalyzer
.NewBuilder()
.HideMatcherLoadStats()
.DelayInitialization()
.Build()
.GetAllPossibleFieldNamesSorted();
foreach (var fieldName in fieldNames)
{
yield return(new object[] { fieldName });
}
}
public void TestUserAgentMaxLengthSetter()
{
var userAgentAnalyzer = UserAgentAnalyzer.NewBuilder().Build();
userAgentAnalyzer.LoadResources(Config.RESOURCES_PATH, "*-tests.yaml");
userAgentAnalyzer.GetUserAgentMaxLength().Should().Be(UserAgentAnalyzerDirect.DEFAULT_USER_AGENT_MAX_LENGTH, "Default user agent max length");
userAgentAnalyzer.SetUserAgentMaxLength(250);
userAgentAnalyzer.GetUserAgentMaxLength().Should().Be(250, "I set that size");
userAgentAnalyzer.SetUserAgentMaxLength(-100);
userAgentAnalyzer.GetUserAgentMaxLength().Should().Be(UserAgentAnalyzerDirect.DEFAULT_USER_AGENT_MAX_LENGTH, "I set incorrect cache size");;
}
/// <summary>
/// I run the the test in the input file and I check that it throws the expected exteption message.
/// </summary>
/// <param name="inputFilename">The inputFilename<see cref="string"/></param>
/// <param name="message">The message<see cref="string"/></param>
private void RunTest(string inputFilename, string message)
{
var uaaB = UserAgentAnalyzer
.NewBuilder()
.DropDefaultResources()
.KeepTests()
.AddResources($"{Config.RESOURCES_PATH}{Path.DirectorySeparatorChar}YamlParsingTests", inputFilename)
.DelayInitialization();
var a = new Action(() => { uaaB.Build(); });
a.Should().Throw <InvalidParserConfigurationException>().Where(e => e.Message.Contains(message));
}
public void MethodInputValidation()
{
var uaa = UserAgentAnalyzer.NewBuilder()
.WithField("AgentClass")
.Build();
var agent = uaa.Parse((string)null);
agent.Should().NotBeNull();
agent.UserAgentString.Should().BeNull();
agent = uaa.Parse((UserAgent)null);
agent.Should().BeNull();
EvilManualUseragentStringHacks.FixIt(null).Should().BeNull();
}
public void CheckAllPossibleFieldsFastSpeed()
{
Skip.If(!Config.ENABLE_PROFILING);
LOG.Info("Create analyzer");
Stopwatch stopwatch = Stopwatch.StartNew();
UserAgentAnalyzer uaa = UserAgentAnalyzer
.NewBuilder()
.KeepTests()
.DelayInitialization()
.Build();
stopwatch.Stop();
long constructMsecs = stopwatch.ElapsedMilliseconds;;
LOG.Info(string.Format("-- Construction time: {0}ms", constructMsecs));
LOG.Info("List fieldnames");
stopwatch.Restart();
foreach (var n in uaa.GetAllPossibleFieldNamesSorted())
{
LOG.Info(n);
}
stopwatch.Stop();
long listFieldNamesMsecs = stopwatch.ElapsedMilliseconds;
LOG.Info(string.Format("-- List fieldnames: {0}ms", listFieldNamesMsecs));
listFieldNamesMsecs.Should().BeLessThan(500, "Just listing the field names should only take a few ms");
LOG.Info("Initializing the datastructures");
stopwatch.Restart();
uaa.InitializeMatchers();
stopwatch.Stop();
long initializeMsecs = stopwatch.ElapsedMilliseconds;
LOG.Info(string.Format("-- Initialization: {0}ms", initializeMsecs));
initializeMsecs.Should().BeGreaterThan(300, "The initialization should take several seconds");
LOG.Info("Preheat");
stopwatch.Restart();
uaa.PreHeat();
stopwatch.Stop();
long preheatMsecs = stopwatch.ElapsedMilliseconds;
LOG.Info(string.Format("-- Preheat : {0}ms", preheatMsecs));
}
public void TestLimitedFieldsDirect()
{
var builder = UserAgentAnalyzer.NewBuilder();
var userAgentAnalyzer = builder
.Preheat(100)
.Preheat()
.HideMatcherLoadStats()
.ShowMatcherLoadStats()
.WithAllFields()
.WithField("DeviceClass")
.WithField("AgentNameVersionMajor")
.WithUserAgentMaxLength(1234)
.Build();
userAgentAnalyzer.GetUserAgentMaxLength().Should().Be(1234);
this.RunTestCase(userAgentAnalyzer);
}
public void TestLoadOnlyCustomRules()
{
var userAgentAnalyzer =
UserAgentAnalyzer
.NewBuilder()
.WithoutCache()
.HideMatcherLoadStats()
.AddResources(Config.RESOURCES_PATH, "ExtraLoadedRule1.yaml")
.WithField("ExtraValue2")
.WithField("ExtraValue1")
.AddResources(Config.RESOURCES_PATH, "ExtraLoadedRule2.yaml")
.Build();
var parsedAgent = userAgentAnalyzer.Parse("Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36");
// The requested fields
parsedAgent.GetValue("ExtraValue1").Should().Be("One");
parsedAgent.GetValue("ExtraValue2").Should().Be("Two");
}
public void TestPreheatNoTests()
{
var userAgentAnalyzer =
UserAgentAnalyzer
.NewBuilder()
.KeepTests()
.HideMatcherLoadStats()
.WithField("AgentName")
.Build();
userAgentAnalyzer.NumberOfTestCases.Should().BeGreaterThan(100);
userAgentAnalyzer.PreHeat(0).Should().Be(0);
userAgentAnalyzer.PreHeat(-1).Should().Be(0);
userAgentAnalyzer.PreHeat(1000000000L).Should().Be(0);
userAgentAnalyzer.DropTests();
userAgentAnalyzer.NumberOfTestCases.Should().Be(0);
userAgentAnalyzer.PreHeat().Should().Be(0);
}
public void TestSettingNoCaching()
{
var uaa = UserAgentAnalyzer
.NewBuilder()
.WithoutCache()
.HideMatcherLoadStats()
.WithField("AgentUuid")
.Build();
uaa.CacheSize.Should().Be(0);
this.GetAllocatedCacheSize(uaa).Should().Be(0);
uaa.SetCacheSize(42);
uaa.CacheSize.Should().Be(42);
this.GetAllocatedCacheSize(uaa).Should().BeGreaterOrEqualTo(42);
uaa.DisableCaching();
uaa.CacheSize.Should().Be(0);
this.GetAllocatedCacheSize(uaa).Should().BeGreaterOrEqualTo(0);
}
public void TestLoadOnlyCompanyCustomFormatRules()
{
var userAgentAnalyzer =
UserAgentAnalyzer
.NewBuilder()
.WithoutCache()
.HideMatcherLoadStats()
.DropDefaultResources()
.AddResources(Config.RESOURCES_PATH, "CompanyInternalUserAgents.yaml")
.WithFields("ApplicationName", "ApplicationVersion")
.WithFields("ApplicationInstance", "ApplicationGitCommit")
.WithField("ServerName")
.Build();
var parsedAgent = userAgentAnalyzer.Parse("TestApplication/1.2.3 (node123.datacenter.example.nl; 1234; d71922715c2bfe29343644b14a4731bf5690e66e)");
// The requested fields
parsedAgent.GetValue("ApplicationName").Should().Be("TestApplication");
parsedAgent.GetValue("ApplicationVersion").Should().Be("1.2.3");
parsedAgent.GetValue("ApplicationInstance").Should().Be("1234");
parsedAgent.GetValue("ApplicationGitCommit").Should().Be("d71922715c2bfe29343644b14a4731bf5690e66e");
parsedAgent.GetValue("ServerName").Should().Be("node123.datacenter.example.nl");
}
/// <summary>
/// This method is an helper that does a little trick for us, it reads private property to get the private cache.
/// this is used to check if the parsed user agent are really cached.
/// </summary>
/// <param name="uaa">The user agent analyzer from wich we want retrieve the private cache.</param>
/// <returns>The cached <see cref="IDictionary<string, UserAgent>"> of parsed user agents.</returns>
private IDictionary <string, UserAgent> GetCache(UserAgentAnalyzer uaa)
{
return(uaa?.ParseCache);
}
