public SecureModuleFeature(ITestOutputHelper output) : base(output)
{
this.output = output;
config = new FeaturesConfig();
user = DataGenerator.GenerateSingleUser(config.Context);
helper = new FeatureHelper(config);
}
public RoleFeature(ITestOutputHelper output) : base(output)
{
config = new FeaturesConfig();
user = DataGenerator.GenerateSingleUser(config.Context);
team = DataGenerator.GenerateSingleTeam(config.Context);
helper = new FeatureHelper(config);
}
public ExactAdmin1AddOnFeaturesGenerator(FeaturesConfig featuresConfig) : base(featuresConfig)
{
if (this.FeaturesConfig.NullDefaultsAllowed)
{
FeatureDefaults = new Features()
{
{ CityFeatureType.ExactAdmin1NameMatch, false },
{ CityFeatureType.ExactAdmin1Letters, (byte?)null }
};
if (this.FeaturesConfig.UseSlotIndex)
{
FeatureDefaults[CityFeatureType.ExactAdmin1RTLSlotIndex] = (byte?)null;
FeatureDefaults[CityFeatureType.ExactAdmin1LTRSlotIndex] = (byte?)null;
}
}
else
{
FeatureDefaults = new Features()
{
{ CityFeatureType.ExactAdmin1NameMatch, false },
{ CityFeatureType.ExactAdmin1Letters, (byte?)0 }
};
if (this.FeaturesConfig.UseSlotIndex)
{
FeatureDefaults[CityFeatureType.ExactAdmin1RTLSlotIndex] = (byte?)byte.MaxValue;
FeatureDefaults[CityFeatureType.ExactAdmin1LTRSlotIndex] = (byte?)byte.MaxValue;
}
}
FeatureDefaultsValueTypes = new FeatureValueTypes()
{
{ CityFeatureType.ExactAdmin1NameMatch, typeof(bool) },
{ CityFeatureType.ExactAdmin1Letters, typeof(byte?) }
};
if (this.FeaturesConfig.UseSlotIndex)
{
FeatureDefaultsValueTypes[CityFeatureType.ExactAdmin1RTLSlotIndex] = typeof(byte?);
FeatureDefaultsValueTypes[CityFeatureType.ExactAdmin1LTRSlotIndex] = typeof(byte?);
}
FeatureGranularities = new FeatureGranularities()
{
{ CityFeatureType.ExactAdmin1NameMatch, FeatureGranularity.Discrete },
{ CityFeatureType.ExactAdmin1Letters, FeatureGranularity.Continuous },
};
if (this.FeaturesConfig.UseSlotIndex)
{
FeatureGranularities[CityFeatureType.ExactAdmin1RTLSlotIndex] = FeatureGranularity.Discrete;
FeatureGranularities[CityFeatureType.ExactAdmin1LTRSlotIndex] = FeatureGranularity.Discrete;
}
}
//private readonly HttpClient _httpClient;
//private ExternalServicesConfig _externalServicesConfig;
//public ProductsController(IProductRepo repository, IMapper mapper, IOptions<FeaturesConfig> options, HttpClient httpClient, IOptionsMonitor<ExternalServicesConfig> externalServicesOptions)
public ProductsController(IProductRepo repository, IMapper mapper, IOptions <FeaturesConfig> options)
{
_repository = repository;
_mapper = mapper;
_featuresConfig = options.Value;
//var externalServicesConfig = externalServicesOptions.Get(ExternalServicesConfig.WeatherApi);
//httpClient.BaseAddress = new Uri(externalServicesConfig.Url);
//_httpClient = httpClient;
}
/// <summary>
/// Process a feature block (jsonobj) and add discovered feature configuration to an existing or new feature object
/// </summary>
/// <param name="featuresConfig"></param>
/// <param name="key"></param>
/// <param name="value"></param>
private void ProcessFeatureItem(FeaturesConfig featuresConfig, string key, JToken jsonobj)
{
var result = featuresConfig.Features.TryGetValue(key, out Feature feature);
if (!result)
{
feature = new Feature
{
Name = key,
Urls = new StringNode
{
Children = new Dictionary <string, StringNode>()
}
};
featuresConfig.Features.Add(key, feature);
}
foreach (var url in (JObject)jsonobj)
{
if (url.Key == PARENTKEY)
{
var parentkey = url.Value.ToString();
bool parentfound = featuresConfig.Features.TryGetValue(parentkey, out Feature parent);
if (!parentfound)
{
logger.LogWarning("Parent feature must be specified above child feature {parent} {child}", parentkey, key);
}
else
{
feature.Parent = parent;
}
continue;
}
var output = ParseHostname(url.Key);
if (output.hostnameEnum == HostnameEnum.NoHostname)
{
// it is a url
// feature.Urls.Add(output.output, url.Value.ToString());
var value = url.Value.ToString();
AddPathToUrls(feature, output.output, value);
}
else
{
// it is a hostname
logger.LogWarning("Hostnames within feature configuration is not allowed {feature} {hostname}", key, output.output);
}
}
}
public TLDAddOnFeaturesGenerator(FeaturesConfig featuresConfig) : base(featuresConfig)
{
FeatureDefaults = new Features()
{
{ CityFeatureType.TLDMatch, false },
};
FeatureDefaultsValueTypes = new FeatureValueTypes()
{
{ CityFeatureType.TLDMatch, typeof(bool) },
};
FeatureGranularities = new FeatureGranularities()
{
{ CityFeatureType.TLDMatch, FeatureGranularity.Discrete }
};
}
public HostnamePatternsAddOnFeaturesGenerator(FeaturesConfig featuresConfig,
HostnamePatternMiner miner,
Dictionary <string, Dictionary <PatternRule, PatternMiningCoordinates> > hostnamePatternRules) : base(featuresConfig)
{
this.miner = miner;
this.hostnamePatternRules = hostnamePatternRules;
FeatureDefaults = new Features()
{
{ CityFeatureType.HostnamePatternMatch, false },
};
FeatureDefaultsValueTypes = new FeatureValueTypes()
{
{ CityFeatureType.HostnamePatternMatch, typeof(bool) },
};
FeatureGranularities = new FeatureGranularities()
{
{ CityFeatureType.HostnamePatternMatch, FeatureGranularity.Discrete }
};
}
private void ProcessRootItem(string key, JToken value)
{
var output = ParseHostname(key);
if (output.hostnameEnum == HostnameEnum.NoHostname)
{
// process a single feature into the list of DefaultHost features
ProcessFeatureItem(featuresRoot.DefaultHost, key, value);
}
else
{
var result = featuresRoot.Hostnames.TryGetValue(output.output, out FeaturesConfig features);
if (!result)
{
features = new FeaturesConfig
{
Features = new Dictionary <string, Feature>()
};
featuresRoot.Hostnames.Add(output.output, features);
}
ProcessFeatures(features, value);
}
}
public HostnamePatternsFeaturesGenerator(
FeaturesConfig featuresConfig,
HostnamePatternMiner miner,
Dictionary <string, Dictionary <PatternRule, PatternMiningCoordinates> > hostnamePatternRules) : base(featuresConfig)
{
this.miner = miner;
this.hostnamePatternRules = hostnamePatternRules;
if (this.FeaturesConfig.NullDefaultsAllowed)
{
FeatureDefaults = new Features()
{
{ CityFeatureType.HostnamePatternMatch, false },
{ CityFeatureType.HostnamePatternConfidence, (float?)null }
};
}
else
{
FeatureDefaults = new Features()
{
{ CityFeatureType.HostnamePatternMatch, false },
{ CityFeatureType.HostnamePatternConfidence, (float?)0 }
};
}
FeatureDefaultsValueTypes = new FeatureValueTypes()
{
{ CityFeatureType.HostnamePatternMatch, typeof(bool) },
{ CityFeatureType.HostnamePatternConfidence, typeof(float?) }
};
FeatureGranularities = new FeatureGranularities()
{
{ CityFeatureType.HostnamePatternMatch, FeatureGranularity.Discrete },
{ CityFeatureType.HostnamePatternConfidence, FeatureGranularity.Continuous }
};
}
public TrainingData Sample(
string citiesPath,
string alternateNamesPath,
string admin1Path,
string admin2Path,
string countriesPath,
string clliPath,
string unlocodePath,
string groundTruthPath,
int trueNegativesMultiplier,
ShouldProcessHostname shouldProcessHostname = null,
ShouldContinueIngestingNewHostnames shouldContinueIngestingNewHostnames = null,
ShowConsoleStats showConsoleStats = null,
FeaturesConfig featuresConfig = null)
{
if (featuresConfig == null)
{
featuresConfig = new FeaturesConfig();
}
var aggregator = new CityFeaturesAggregator(citiesPath, alternateNamesPath, admin1Path, admin2Path, countriesPath, clliPath, unlocodePath, featuresConfig: featuresConfig);
this.Aggregator = aggregator;
var trainingData = new TrainingData(tableName: "ReverseDNSGeolocation Training", featuresAggregator: aggregator);
this.TrainingData = trainingData;
string line;
var counter = 0;
var storedTruePositivesCount = 0;
var storedTrueNegativesCount = 0;
var positivesFeaturesDistribution = new Dictionary <CityFeatureType, int>();
var rand = new Random();
using (var file = new StreamReader(groundTruthPath))
{
while ((line = file.ReadLine()) != null)
{
if (string.IsNullOrWhiteSpace(line) ||
(line.Length > 0 && line[0] == '#'))
{
continue;
}
var parts = new List <string>(line.Split(new char[] { '\t' }));
if (parts.Count != 13)
{
continue;
}
/*
* 0 RawIP string
* 1 NumericIP uint
* 2 Bucket uint?
* 3 ReverseDNSHostname string
* 4 RealtimeLatitude double?
* 5 RealtimeLongitude double?
* 6 RealtimeCountryISO string
* 7 RealtimeCountryConfidence byte
* 8 RealtimeState string
* 9 RealtimeStateConfidence byte
* 10 RealtimeCity string
* 11 RealtimeCityConfidence byte
* 12 RealtimeAccuracyKm double?
*/
var hostname = parts[3];
var trueLatitudeStr = parts[4];
var trueLongitudeStr = parts[5];
if (string.IsNullOrWhiteSpace(hostname) ||
string.IsNullOrWhiteSpace(trueLatitudeStr) ||
string.IsNullOrWhiteSpace(trueLongitudeStr))
{
continue;
}
double trueLatitude;
double trueLongitude;
if (!double.TryParse(trueLatitudeStr, out trueLatitude) ||
!double.TryParse(trueLongitudeStr, out trueLongitude))
{
continue;
}
var parsedHostname = HostnameSplitter.Split(hostname);
if (shouldProcessHostname != null && !shouldProcessHostname(hostname, parsedHostname, storedTruePositivesCount, storedTrueNegativesCount))
{
continue;
}
counter++;
if (showConsoleStats != null)
{
showConsoleStats(hostname, parsedHostname, storedTruePositivesCount, storedTrueNegativesCount);
}
var candidatesAndFeatures = aggregator.GenerateCandidatesForHostname(parsedHostname);
var truesPositives = new List <DataRow>();
var trueNegatives = new List <DataRow>();
foreach (var candidateEntry in candidatesAndFeatures)
{
var locationCandidate = candidateEntry.Key;
var locationFeatures = candidateEntry.Value;
var distance = DistanceHelper.Distance(trueLatitude, trueLongitude, locationCandidate.Latitude, locationCandidate.Longitude, DistanceUnit.Mile);
if (distance <= featuresConfig.TruePositiveMaximumDistanceKilometers)
{
if (this.ShouldAddPositiveExample(positivesFeaturesDistribution, locationFeatures))
{
var newRow = trainingData.CreateTrainingRow(locationFeatures, isValidLocation: true);
truesPositives.Add(newRow);
this.AddToFeaturesDistribution(positivesFeaturesDistribution, locationFeatures);
}
/*
* if (storedTruePositivesCount % 100 == 0)
* {
* Console.WriteLine($"{hostname} - {locationCandidate}");
* Console.WriteLine("---");
*
* foreach (var feature in locationFeatures)
* {
* Console.WriteLine($"{feature.Key} = {feature.Value}");
* }
*
* Console.WriteLine("---------------------------------");
* }
*/
/*
* if ((bool)locationFeatures[CityFeatureType.ExactAdmin1NameMatch] == true && (bool)locationFeatures[CityFeatureType.CityAdmin1NameMatch] == true)
* {
* Console.WriteLine($"{hostname} - {locationCandidate}");
* Console.WriteLine("---");
*
* foreach (var feature in locationFeatures)
* {
* Console.WriteLine($"{feature.Key} = {feature.Value}");
* }
*
* Console.WriteLine("---------------------------------");
* }
*/
}
else
{
var newRow = trainingData.CreateTrainingRow(locationFeatures, isValidLocation: false);
trueNegatives.Add(newRow);
/*
* if ((bool)locationFeatures[CityFeatureType.ExactAdmin1NameMatch] == true && (bool)locationFeatures[CityFeatureType.CityAdmin1NameMatch] == true)
* {
* Console.WriteLine($"{hostname} - {locationCandidate}");
* Console.WriteLine("---");
*
* foreach (var feature in locationFeatures)
* {
* Console.WriteLine($"{feature.Key} = {feature.Value}");
* }
*
* Console.WriteLine("---------------------------------");
* }
*/
}
}
// WARNING: Do not move this above the true negatives selection, at it will bias the data (even more)
if (truesPositives.Count > 0)
{
truesPositives.ForEach(tp => trainingData.AddTrainingRow(tp));
storedTruePositivesCount += truesPositives.Count;
}
var neededTrueNegativeItemsCount = storedTruePositivesCount * trueNegativesMultiplier;
if (trueNegatives.Count > 0 && storedTrueNegativesCount < neededTrueNegativeItemsCount)
{
var neededItemsCount = 0;
if (truesPositives.Count > 0)
{
neededItemsCount = Math.Min(truesPositives.Count, Math.Min(trueNegatives.Count, neededTrueNegativeItemsCount));
}
else
{
neededItemsCount = Math.Min(trueNegatives.Count, neededTrueNegativeItemsCount);
}
var extractedRandTrueNegativeItems = trueNegatives.OrderBy(x => rand.Next()).Take(neededItemsCount);
foreach (var trueNegativeItem in extractedRandTrueNegativeItems)
{
trainingData.AddTrainingRow(trueNegativeItem);
storedTrueNegativesCount++;
}
}
if (counter % 1000 == 0)
{
Console.WriteLine("------------------------------------");
foreach (var entry in positivesFeaturesDistribution)
{
Console.WriteLine($"Positive: {entry.Key}\t{entry.Value}");
}
Console.WriteLine("------------------------------------");
}
if (shouldContinueIngestingNewHostnames != null && !shouldContinueIngestingNewHostnames(hostname, parsedHostname, storedTruePositivesCount, storedTrueNegativesCount))
{
showConsoleStats(hostname, parsedHostname, storedTruePositivesCount, storedTrueNegativesCount, lastRow: true);
break;
}
}
}
trainingData.FinalizeData();
return(trainingData);
}
public FeatureHelper(FeaturesConfig config)
{
this.config = config;
}
public AddOnCityFeaturesGenerator(FeaturesConfig featuresConfig)
{
this.FeaturesConfig = featuresConfig;
}
