public override void AppendFeatures(HostnameSplitterResult parsedHostname, GeonamesCityEntity cityEntity, Features features) { if (parsedHostname?.SubdomainParts == null || cityEntity == null || cityEntity.CountryEntity == null || string.IsNullOrWhiteSpace(cityEntity.CountryEntity.Name) || features == null) { return; } var countryNameVariations = cityEntity.CountryEntity.NameVariationsLower; foreach (var subdomainPart in parsedHostname.SubdomainParts) { if (countryNameVariations.Contains(subdomainPart.Substring)) { features[CityFeatureType.ExactCountryNameMatch] = true; if (!features.ContainsKey(CityFeatureType.ExactCountryLetters) || features[CityFeatureType.ExactCountryLetters] == null || ((byte)features[CityFeatureType.ExactCountryLetters]) < subdomainPart.Substring.Length) { features[CityFeatureType.ExactCountryLetters] = Convert.ToByte(subdomainPart.Substring.Length); } if (this.FeaturesConfig.UseSlotIndex) { features[CityFeatureType.ExactCountryRTLSlotIndex] = subdomainPart.RTLSlotIndex; features[CityFeatureType.ExactCountryLTRSlotIndex] = subdomainPart.LTRSlotIndex; } } } }
public override void AppendFeatures(HostnameSplitterResult parsedHostname, GeonamesCityEntity cityEntity, Features features) { if (parsedHostname == null || string.IsNullOrWhiteSpace(parsedHostname.TLD) || cityEntity == null || cityEntity.CountryEntity == null || string.IsNullOrWhiteSpace(cityEntity.CountryEntity.TLD) || features == null) { return; } var countryTld = cityEntity.CountryEntity.TLD.ToLowerInvariant(); var hostnameTLD = parsedHostname.TLD; var hostnameTLDParts = hostnameTLD.Split('.'); var lastPart = hostnameTLDParts[hostnameTLDParts.Length - 1]; if (!string.IsNullOrWhiteSpace(lastPart)) { string convertedTld; var hostnameTld = string.Format(CultureInfo.InvariantCulture, ".{0}", lastPart.ToLowerInvariant()); if (countryTld == hostnameTld) { features[CityFeatureType.TLDMatch] = true; } else if ( parsedHostname.DomainInfo?.RegistrableDomain != null && DomainsToCountryTlds.TryGetValue(parsedHostname.DomainInfo.RegistrableDomain, out convertedTld) && countryTld == convertedTld) { features[CityFeatureType.TLDMatch] = true; } } }
public override void AppendFeatures(HostnameSplitterResult parsedHostname, GeonamesCityEntity cityEntity, Features features) { if (parsedHostname?.SubdomainParts == null || cityEntity == null || cityEntity.Admin1Entity == null || string.IsNullOrWhiteSpace(cityEntity.Admin1Entity.Name) || features == null) { return; } var firstLettersAdmin1NameVariations = this.GenerateVariationsForName(cityEntity.Admin1Entity.Name); foreach (var subdomainPart in parsedHostname.SubdomainParts) { if (firstLettersAdmin1NameVariations.Contains(subdomainPart.Substring)) { features[CityFeatureType.FirstLettersAdmin1NameMatch] = true; if (!features.ContainsKey(CityFeatureType.FirstLettersAdmin1Letters) || features[CityFeatureType.FirstLettersAdmin1Letters] == null || ((byte)features[CityFeatureType.FirstLettersAdmin1Letters]) < subdomainPart.Substring.Length) { features[CityFeatureType.FirstLettersAdmin1Letters] = Convert.ToByte(subdomainPart.Substring.Length); features[CityFeatureType.FirstLettersAdmin1LettersRatio] = (float?)(subdomainPart.Substring.Length / ((1.0f) * cityEntity.Admin1Entity.Name.Length)); } if (this.FeaturesConfig.UseSlotIndex) { features[CityFeatureType.FirstLettersAdmin1RTLSlotIndex] = subdomainPart.RTLSlotIndex; features[CityFeatureType.FirstLettersAdmin1LTRSlotIndex] = subdomainPart.LTRSlotIndex; } } } }
private bool ShouldAcceptDomain(string hostname, HostnameSplitterResult parsedHostname) { if (parsedHostname?.DomainInfo?.RegistrableDomain == null || string.IsNullOrEmpty(hostname) || !hostname.Contains(".") || hostname.Contains(" ") || hostname.Contains(",")) { return(false); } var domain = parsedHostname.DomainInfo.RegistrableDomain; /* * if (hostname.ToLowerInvariant().Contains("comcast.net")) * { * return false; * } */ int currentHostnameCount; if (this.HostnameCounts.TryGetValue(domain, out currentHostnameCount)) { return(currentHostnameCount <= this.maxPositiveSamplesPerDomain); } else { return(true); } }
public override void AppendFeatures(HostnameSplitterResult parsedHostname, GeonamesCityEntity cityEntity, Features features) { if (parsedHostname == null || parsedHostname.DomainInfo?.RegistrableDomain == null || parsedHostname.SubdomainParts == null || parsedHostname.SubdomainParts.Count == 0 || cityEntity == null) { return; } var domain = parsedHostname.DomainInfo.RegistrableDomain; Dictionary <PatternRule, PatternMiningCoordinates> rulesToCoordinates; if (!this.hostnamePatternRules.TryGetValue(domain, out rulesToCoordinates)) { return; } var subdomainParts = parsedHostname.SubdomainParts; if (subdomainParts == null || subdomainParts.Count == 0) { return; } var ruleAtoms = this.miner.CreateRuleAtoms(subdomainParts); if (ruleAtoms == null || ruleAtoms.Count == 0) { return; } var rules = this.miner.GeneratePossibleRules(ruleAtoms); if (rules == null || rules.Count == 0) { return; } foreach (var rule in rules) { PatternMiningCoordinates coordinates; if (rulesToCoordinates.TryGetValue(rule, out coordinates)) { var distance = DistanceHelper.Distance(cityEntity.Latitude, cityEntity.Longitude, coordinates.Latitude, coordinates.Longitude, DistanceUnit.Kilometer); // TODO: Make this configurable // TODO: Distance should vary depending on geohash length? if (distance <= 100) { features[CityFeatureType.HostnamePatternMatch] = true; } } } }
private bool ShouldAcceptDomain(string hostname, HostnameSplitterResult parsedHostname) { if (parsedHostname?.DomainInfo?.RegistrableDomain == null || string.IsNullOrEmpty(hostname) || !hostname.Contains(".") || hostname.Contains(" ") || hostname.Contains(",")) { return(false); } return(true); }
private bool ShouldContinueIngestingNewHostnames(string hostname, HostnameSplitterResult parsedHostname, int storedTruePositivesCount, int storedTrueNegativesCount) { ///if (this.ValidLinesCounter > 10000000) /* * if (this.ValidLinesCounter > 30000000) * { * return false; * } */ return(true); }
public override void AppendFeatures(HostnameSplitterResult parsedHostname, GeonamesCityEntity cityEntity, Features features) { if (!this.FeaturesConfig.UseDomainAsFeature) { return; } if (parsedHostname?.DomainInfo?.RegistrableDomain == null || features == null) { return; } var domain = parsedHostname.DomainInfo.RegistrableDomain; features[CityFeatureType.Domain] = domain.GetHashCode(); }
private void IncrementHostnameCount(HostnameSplitterResult parsedHostname) { var domain = parsedHostname?.DomainInfo?.RegistrableDomain; if (string.IsNullOrWhiteSpace(domain)) { return; } int currentHostnameCount; if (!this.HostnameCounts.TryGetValue(domain, out currentHostnameCount)) { currentHostnameCount = 0; } this.HostnameCounts[domain] = currentHostnameCount + 1; }
public abstract Dictionary <GeonamesCityEntity, Features> GenerateCandidatesAndFeatures(HostnameSplitterResult parsedHostname);
public override Dictionary <GeonamesCityEntity, Features> GenerateCandidatesAndFeatures(HostnameSplitterResult parsedHostname) { var candidatesAndFeatures = new Dictionary <GeonamesCityEntity, Features>(); var combinedParts = new HashSet <SubdomainPart>(); if (parsedHostname?.SubdomainParts != null) { combinedParts.UnionWith(parsedHostname.SubdomainParts); } foreach (var subdomainPart in combinedParts) { EntitiesToFeatures entitiesToFeatures; if (codesToEntitiesToFeatures.TryGetValue(subdomainPart.Substring, out entitiesToFeatures)) { foreach (var entry in entitiesToFeatures) { var features = new Features(); foreach (var featureEntry in entry.Value) { features[featureEntry.Key] = featureEntry.Value; } if (this.FeaturesConfig.UseSlotIndex) { features[CityFeatureType.ExactUNLOCODECodeRTLSlotIndex] = subdomainPart.RTLSlotIndex; features[CityFeatureType.ExactUNLOCODECodeLTRSlotIndex] = subdomainPart.LTRSlotIndex; } candidatesAndFeatures[entry.Key] = features; } } } return(candidatesAndFeatures); }
public Dictionary <GeonamesCityEntity, Features> GenerateCandidatesForHostname(HostnameSplitterResult parsedHostname) { this.TotalHostnamesSeen++; var candidatesAndFeatures = new Dictionary <GeonamesCityEntity, Features>(); foreach (var cityFeatureGenerator in this.cityFeatureGenerators) { var stopwatchName = cityFeatureGenerator.GetType().ToString(); PrimaryFeatureStopWatches[stopwatchName].Start(); var localCandidatesAndFeatures = cityFeatureGenerator.GenerateCandidatesAndFeatures(parsedHostname); PrimaryFeatureStopWatches[stopwatchName].Stop(); if (localCandidatesAndFeatures.Count > 0) { FeatureHostnameHitCounts[stopwatchName]++; } foreach (var localCandidatesAndFeaturesEntry in localCandidatesAndFeatures) { var cityEntity = localCandidatesAndFeaturesEntry.Key; var localFeatures = localCandidatesAndFeaturesEntry.Value; Features globalFeatures; if (!candidatesAndFeatures.TryGetValue(cityEntity, out globalFeatures)) { globalFeatures = new Features(); candidatesAndFeatures[cityEntity] = globalFeatures; } foreach (var localFeatureEntry in localFeatures) { globalFeatures[localFeatureEntry.Key] = localFeatureEntry.Value; } } } foreach (var candidatesAndFeaturesEntry in candidatesAndFeatures) { var cityEntity = candidatesAndFeaturesEntry.Key; var features = candidatesAndFeaturesEntry.Value; foreach (var addOnGenerator in this.cityAddOnFeatureGenerators) { this.SecondaryFeaturesStopWatch.Start(); addOnGenerator.AppendFeatures(parsedHostname, cityEntity, features); this.SecondaryFeaturesStopWatch.Stop(); } } // We need to add the defaults outside of the feature generation because // if a feature does not match at all, we might still want to fill in the defaults if (this.featuresConfig.InitializeDefaultFeatures) { foreach (var candidatesAndFeaturesEntry in candidatesAndFeatures) { var features = candidatesAndFeaturesEntry.Value; foreach (var cityFeatureGenerator in cityFeatureGenerators) { var defaultFeatures = cityFeatureGenerator.FeatureDefaults; foreach (var defaultFeaturesEntry in defaultFeatures) { if (!features.ContainsKey(defaultFeaturesEntry.Key)) { features[defaultFeaturesEntry.Key] = defaultFeaturesEntry.Value; } } } foreach (var addOnGenerator in this.cityAddOnFeatureGenerators) { var defaultAddOnFeatures = addOnGenerator.FeatureDefaults; foreach (var defaultAddOnFeaturesEntry in defaultAddOnFeatures) { if (!features.ContainsKey(defaultAddOnFeaturesEntry.Key)) { features[defaultAddOnFeaturesEntry.Key] = defaultAddOnFeaturesEntry.Value; } } } } } TotalCandidatesSeen += candidatesAndFeatures.Count; return(candidatesAndFeatures); }
public override Dictionary <GeonamesCityEntity, Features> GenerateCandidatesAndFeatures(HostnameSplitterResult parsedHostname) { var candidatesAndFeatures = new Dictionary <GeonamesCityEntity, Features>(); if (parsedHostname?.SubdomainParts == null) { return(candidatesAndFeatures); } foreach (var subdomainPart in parsedHostname.SubdomainParts) { EntitiesToFeatures entitiesToFeatures; if (variationsToEntitiesToFeatures.TryGetValue(subdomainPart.Substring, out entitiesToFeatures)) { foreach (var entry in entitiesToFeatures) { var features = new Features(); foreach (var featureEntry in entry.Value) { features[featureEntry.Key] = featureEntry.Value; } if (this.FeaturesConfig.UseSlotIndex) { features[CityFeatureType.AlternateCityNameRTLSlotIndex] = subdomainPart.RTLSlotIndex; features[CityFeatureType.AlternateCityNameLTRSlotIndex] = subdomainPart.LTRSlotIndex; } candidatesAndFeatures[entry.Key] = features; } } } return(candidatesAndFeatures); }
public abstract void AppendFeatures(HostnameSplitterResult parsedHostname, GeonamesCityEntity cityEntity, Features features);
public override Dictionary <GeonamesCityEntity, Features> GenerateCandidatesAndFeatures(HostnameSplitterResult parsedHostname) { var candidatesAndFeatures = new Dictionary <GeonamesCityEntity, Features>(); var domain = parsedHostname?.DomainInfo?.RegistrableDomain; var subdomainParts = parsedHostname.SubdomainParts; if (subdomainParts == null || subdomainParts.Count == 0) { return(candidatesAndFeatures); } var ruleAtoms = this.miner.CreateRuleAtoms(subdomainParts); var rules = this.miner.GeneratePossibleRules(ruleAtoms); if (rules == null || rules.Count == 0) { return(candidatesAndFeatures); } PatternMiningCoordinates bestCoordinates = null; Dictionary <PatternRule, PatternMiningCoordinates> rulesToCoordinates; if (this.hostnamePatternRules.TryGetValue(domain, out rulesToCoordinates)) { foreach (var rule in rules) { PatternMiningCoordinates currentCoordinates; if (rulesToCoordinates.TryGetValue(rule, out currentCoordinates)) { if (currentCoordinates.ClosestCity != null) { if (bestCoordinates == null || currentCoordinates.Confidence > bestCoordinates.Confidence) { bestCoordinates = currentCoordinates; } } } } } if (bestCoordinates != null) { var features = this.InitializeDefaultFeatureValues(); features[CityFeatureType.HostnamePatternMatch] = true; features[CityFeatureType.HostnamePatternConfidence] = bestCoordinates.Confidence; candidatesAndFeatures[bestCoordinates.ClosestCity] = features; } return(candidatesAndFeatures); }