public SecureModuleFeature(ITestOutputHelper output) : base(output) { this.output = output; config = new FeaturesConfig(); user = DataGenerator.GenerateSingleUser(config.Context); helper = new FeatureHelper(config); }
public RoleFeature(ITestOutputHelper output) : base(output) { config = new FeaturesConfig(); user = DataGenerator.GenerateSingleUser(config.Context); team = DataGenerator.GenerateSingleTeam(config.Context); helper = new FeatureHelper(config); }
public ExactAdmin1AddOnFeaturesGenerator(FeaturesConfig featuresConfig) : base(featuresConfig) { if (this.FeaturesConfig.NullDefaultsAllowed) { FeatureDefaults = new Features() { { CityFeatureType.ExactAdmin1NameMatch, false }, { CityFeatureType.ExactAdmin1Letters, (byte?)null } }; if (this.FeaturesConfig.UseSlotIndex) { FeatureDefaults[CityFeatureType.ExactAdmin1RTLSlotIndex] = (byte?)null; FeatureDefaults[CityFeatureType.ExactAdmin1LTRSlotIndex] = (byte?)null; } } else { FeatureDefaults = new Features() { { CityFeatureType.ExactAdmin1NameMatch, false }, { CityFeatureType.ExactAdmin1Letters, (byte?)0 } }; if (this.FeaturesConfig.UseSlotIndex) { FeatureDefaults[CityFeatureType.ExactAdmin1RTLSlotIndex] = (byte?)byte.MaxValue; FeatureDefaults[CityFeatureType.ExactAdmin1LTRSlotIndex] = (byte?)byte.MaxValue; } } FeatureDefaultsValueTypes = new FeatureValueTypes() { { CityFeatureType.ExactAdmin1NameMatch, typeof(bool) }, { CityFeatureType.ExactAdmin1Letters, typeof(byte?) } }; if (this.FeaturesConfig.UseSlotIndex) { FeatureDefaultsValueTypes[CityFeatureType.ExactAdmin1RTLSlotIndex] = typeof(byte?); FeatureDefaultsValueTypes[CityFeatureType.ExactAdmin1LTRSlotIndex] = typeof(byte?); } FeatureGranularities = new FeatureGranularities() { { CityFeatureType.ExactAdmin1NameMatch, FeatureGranularity.Discrete }, { CityFeatureType.ExactAdmin1Letters, FeatureGranularity.Continuous }, }; if (this.FeaturesConfig.UseSlotIndex) { FeatureGranularities[CityFeatureType.ExactAdmin1RTLSlotIndex] = FeatureGranularity.Discrete; FeatureGranularities[CityFeatureType.ExactAdmin1LTRSlotIndex] = FeatureGranularity.Discrete; } }
//private readonly HttpClient _httpClient; //private ExternalServicesConfig _externalServicesConfig; //public ProductsController(IProductRepo repository, IMapper mapper, IOptions<FeaturesConfig> options, HttpClient httpClient, IOptionsMonitor<ExternalServicesConfig> externalServicesOptions) public ProductsController(IProductRepo repository, IMapper mapper, IOptions <FeaturesConfig> options) { _repository = repository; _mapper = mapper; _featuresConfig = options.Value; //var externalServicesConfig = externalServicesOptions.Get(ExternalServicesConfig.WeatherApi); //httpClient.BaseAddress = new Uri(externalServicesConfig.Url); //_httpClient = httpClient; }
/// <summary> /// Process a feature block (jsonobj) and add discovered feature configuration to an existing or new feature object /// </summary> /// <param name="featuresConfig"></param> /// <param name="key"></param> /// <param name="value"></param> private void ProcessFeatureItem(FeaturesConfig featuresConfig, string key, JToken jsonobj) { var result = featuresConfig.Features.TryGetValue(key, out Feature feature); if (!result) { feature = new Feature { Name = key, Urls = new StringNode { Children = new Dictionary <string, StringNode>() } }; featuresConfig.Features.Add(key, feature); } foreach (var url in (JObject)jsonobj) { if (url.Key == PARENTKEY) { var parentkey = url.Value.ToString(); bool parentfound = featuresConfig.Features.TryGetValue(parentkey, out Feature parent); if (!parentfound) { logger.LogWarning("Parent feature must be specified above child feature {parent} {child}", parentkey, key); } else { feature.Parent = parent; } continue; } var output = ParseHostname(url.Key); if (output.hostnameEnum == HostnameEnum.NoHostname) { // it is a url // feature.Urls.Add(output.output, url.Value.ToString()); var value = url.Value.ToString(); AddPathToUrls(feature, output.output, value); } else { // it is a hostname logger.LogWarning("Hostnames within feature configuration is not allowed {feature} {hostname}", key, output.output); } } }
public TLDAddOnFeaturesGenerator(FeaturesConfig featuresConfig) : base(featuresConfig) { FeatureDefaults = new Features() { { CityFeatureType.TLDMatch, false }, }; FeatureDefaultsValueTypes = new FeatureValueTypes() { { CityFeatureType.TLDMatch, typeof(bool) }, }; FeatureGranularities = new FeatureGranularities() { { CityFeatureType.TLDMatch, FeatureGranularity.Discrete } }; }
public HostnamePatternsAddOnFeaturesGenerator(FeaturesConfig featuresConfig, HostnamePatternMiner miner, Dictionary <string, Dictionary <PatternRule, PatternMiningCoordinates> > hostnamePatternRules) : base(featuresConfig) { this.miner = miner; this.hostnamePatternRules = hostnamePatternRules; FeatureDefaults = new Features() { { CityFeatureType.HostnamePatternMatch, false }, }; FeatureDefaultsValueTypes = new FeatureValueTypes() { { CityFeatureType.HostnamePatternMatch, typeof(bool) }, }; FeatureGranularities = new FeatureGranularities() { { CityFeatureType.HostnamePatternMatch, FeatureGranularity.Discrete } }; }
private void ProcessRootItem(string key, JToken value) { var output = ParseHostname(key); if (output.hostnameEnum == HostnameEnum.NoHostname) { // process a single feature into the list of DefaultHost features ProcessFeatureItem(featuresRoot.DefaultHost, key, value); } else { var result = featuresRoot.Hostnames.TryGetValue(output.output, out FeaturesConfig features); if (!result) { features = new FeaturesConfig { Features = new Dictionary <string, Feature>() }; featuresRoot.Hostnames.Add(output.output, features); } ProcessFeatures(features, value); } }
public HostnamePatternsFeaturesGenerator( FeaturesConfig featuresConfig, HostnamePatternMiner miner, Dictionary <string, Dictionary <PatternRule, PatternMiningCoordinates> > hostnamePatternRules) : base(featuresConfig) { this.miner = miner; this.hostnamePatternRules = hostnamePatternRules; if (this.FeaturesConfig.NullDefaultsAllowed) { FeatureDefaults = new Features() { { CityFeatureType.HostnamePatternMatch, false }, { CityFeatureType.HostnamePatternConfidence, (float?)null } }; } else { FeatureDefaults = new Features() { { CityFeatureType.HostnamePatternMatch, false }, { CityFeatureType.HostnamePatternConfidence, (float?)0 } }; } FeatureDefaultsValueTypes = new FeatureValueTypes() { { CityFeatureType.HostnamePatternMatch, typeof(bool) }, { CityFeatureType.HostnamePatternConfidence, typeof(float?) } }; FeatureGranularities = new FeatureGranularities() { { CityFeatureType.HostnamePatternMatch, FeatureGranularity.Discrete }, { CityFeatureType.HostnamePatternConfidence, FeatureGranularity.Continuous } }; }
public TrainingData Sample( string citiesPath, string alternateNamesPath, string admin1Path, string admin2Path, string countriesPath, string clliPath, string unlocodePath, string groundTruthPath, int trueNegativesMultiplier, ShouldProcessHostname shouldProcessHostname = null, ShouldContinueIngestingNewHostnames shouldContinueIngestingNewHostnames = null, ShowConsoleStats showConsoleStats = null, FeaturesConfig featuresConfig = null) { if (featuresConfig == null) { featuresConfig = new FeaturesConfig(); } var aggregator = new CityFeaturesAggregator(citiesPath, alternateNamesPath, admin1Path, admin2Path, countriesPath, clliPath, unlocodePath, featuresConfig: featuresConfig); this.Aggregator = aggregator; var trainingData = new TrainingData(tableName: "ReverseDNSGeolocation Training", featuresAggregator: aggregator); this.TrainingData = trainingData; string line; var counter = 0; var storedTruePositivesCount = 0; var storedTrueNegativesCount = 0; var positivesFeaturesDistribution = new Dictionary <CityFeatureType, int>(); var rand = new Random(); using (var file = new StreamReader(groundTruthPath)) { while ((line = file.ReadLine()) != null) { if (string.IsNullOrWhiteSpace(line) || (line.Length > 0 && line[0] == '#')) { continue; } var parts = new List <string>(line.Split(new char[] { '\t' })); if (parts.Count != 13) { continue; } /* * 0 RawIP string * 1 NumericIP uint * 2 Bucket uint? * 3 ReverseDNSHostname string * 4 RealtimeLatitude double? * 5 RealtimeLongitude double? * 6 RealtimeCountryISO string * 7 RealtimeCountryConfidence byte * 8 RealtimeState string * 9 RealtimeStateConfidence byte * 10 RealtimeCity string * 11 RealtimeCityConfidence byte * 12 RealtimeAccuracyKm double? */ var hostname = parts[3]; var trueLatitudeStr = parts[4]; var trueLongitudeStr = parts[5]; if (string.IsNullOrWhiteSpace(hostname) || string.IsNullOrWhiteSpace(trueLatitudeStr) || string.IsNullOrWhiteSpace(trueLongitudeStr)) { continue; } double trueLatitude; double trueLongitude; if (!double.TryParse(trueLatitudeStr, out trueLatitude) || !double.TryParse(trueLongitudeStr, out trueLongitude)) { continue; } var parsedHostname = HostnameSplitter.Split(hostname); if (shouldProcessHostname != null && !shouldProcessHostname(hostname, parsedHostname, storedTruePositivesCount, storedTrueNegativesCount)) { continue; } counter++; if (showConsoleStats != null) { showConsoleStats(hostname, parsedHostname, storedTruePositivesCount, storedTrueNegativesCount); } var candidatesAndFeatures = aggregator.GenerateCandidatesForHostname(parsedHostname); var truesPositives = new List <DataRow>(); var trueNegatives = new List <DataRow>(); foreach (var candidateEntry in candidatesAndFeatures) { var locationCandidate = candidateEntry.Key; var locationFeatures = candidateEntry.Value; var distance = DistanceHelper.Distance(trueLatitude, trueLongitude, locationCandidate.Latitude, locationCandidate.Longitude, DistanceUnit.Mile); if (distance <= featuresConfig.TruePositiveMaximumDistanceKilometers) { if (this.ShouldAddPositiveExample(positivesFeaturesDistribution, locationFeatures)) { var newRow = trainingData.CreateTrainingRow(locationFeatures, isValidLocation: true); truesPositives.Add(newRow); this.AddToFeaturesDistribution(positivesFeaturesDistribution, locationFeatures); } /* * if (storedTruePositivesCount % 100 == 0) * { * Console.WriteLine($"{hostname} - {locationCandidate}"); * Console.WriteLine("---"); * * foreach (var feature in locationFeatures) * { * Console.WriteLine($"{feature.Key} = {feature.Value}"); * } * * Console.WriteLine("---------------------------------"); * } */ /* * if ((bool)locationFeatures[CityFeatureType.ExactAdmin1NameMatch] == true && (bool)locationFeatures[CityFeatureType.CityAdmin1NameMatch] == true) * { * Console.WriteLine($"{hostname} - {locationCandidate}"); * Console.WriteLine("---"); * * foreach (var feature in locationFeatures) * { * Console.WriteLine($"{feature.Key} = {feature.Value}"); * } * * Console.WriteLine("---------------------------------"); * } */ } else { var newRow = trainingData.CreateTrainingRow(locationFeatures, isValidLocation: false); trueNegatives.Add(newRow); /* * if ((bool)locationFeatures[CityFeatureType.ExactAdmin1NameMatch] == true && (bool)locationFeatures[CityFeatureType.CityAdmin1NameMatch] == true) * { * Console.WriteLine($"{hostname} - {locationCandidate}"); * Console.WriteLine("---"); * * foreach (var feature in locationFeatures) * { * Console.WriteLine($"{feature.Key} = {feature.Value}"); * } * * Console.WriteLine("---------------------------------"); * } */ } } // WARNING: Do not move this above the true negatives selection, at it will bias the data (even more) if (truesPositives.Count > 0) { truesPositives.ForEach(tp => trainingData.AddTrainingRow(tp)); storedTruePositivesCount += truesPositives.Count; } var neededTrueNegativeItemsCount = storedTruePositivesCount * trueNegativesMultiplier; if (trueNegatives.Count > 0 && storedTrueNegativesCount < neededTrueNegativeItemsCount) { var neededItemsCount = 0; if (truesPositives.Count > 0) { neededItemsCount = Math.Min(truesPositives.Count, Math.Min(trueNegatives.Count, neededTrueNegativeItemsCount)); } else { neededItemsCount = Math.Min(trueNegatives.Count, neededTrueNegativeItemsCount); } var extractedRandTrueNegativeItems = trueNegatives.OrderBy(x => rand.Next()).Take(neededItemsCount); foreach (var trueNegativeItem in extractedRandTrueNegativeItems) { trainingData.AddTrainingRow(trueNegativeItem); storedTrueNegativesCount++; } } if (counter % 1000 == 0) { Console.WriteLine("------------------------------------"); foreach (var entry in positivesFeaturesDistribution) { Console.WriteLine($"Positive: {entry.Key}\t{entry.Value}"); } Console.WriteLine("------------------------------------"); } if (shouldContinueIngestingNewHostnames != null && !shouldContinueIngestingNewHostnames(hostname, parsedHostname, storedTruePositivesCount, storedTrueNegativesCount)) { showConsoleStats(hostname, parsedHostname, storedTruePositivesCount, storedTrueNegativesCount, lastRow: true); break; } } } trainingData.FinalizeData(); return(trainingData); }
public FeatureHelper(FeaturesConfig config) { this.config = config; }
public AddOnCityFeaturesGenerator(FeaturesConfig featuresConfig) { this.FeaturesConfig = featuresConfig; }