private static void FilterGeographicEntity(XElement xml)
{
bool isPlace = xml.Attribute(BlockAttribute.TYPE).IsEqual(EntityType.PlaceName);
bool isUnknown = xml.Attribute(BlockAttribute.TYPE).IsEqual(EntityType.UnknownProper) &&
xml.IsContainAttribute(BlockAttribute.PLACE);
if (isPlace || isUnknown)
{
if (_geoNames != null)
{
string value = xml.Attribute(BlockAttribute.VALUE);
if (value.IsEmpty() || ContainsWrongSymbols(value))
{
//xml.SetType(EntityType.UnknownProper);
xml.SetType(EntityType.PseudoGeo);
return;
}
string valueForTest = value;
string[] words = value.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string word in words)
{
if (Char.IsUpper(word[0]))
{
break;
}
else
{
valueForTest = valueForTest.Substring(word.Length, valueForTest.Length - word.Length).Trim();
}
}
if (!valueForTest.IsEmpty())
{
bool isValid = _geoNames.IsExists(valueForTest);
/*if (!isValid && isPlace)
* {
* xml.SetType(EntityType.UnknownProper);
* }*/
if (!isValid)
{
xml.SetType(EntityType.PseudoGeo);
}
else if (/*isValid &&*/ isUnknown)
{
xml.SetType(EntityType.PlaceName);
xml.SetAttributeValue(BlockAttribute.INFO, "");
if (valueForTest != value)
{
xml.SetAttributeValue(BlockAttribute.VALUE, valueForTest);
}
}
}
}
}
}
public static void IsExists1(this HashtableSearcher searcher, string geoName, int max_len)
{
try
{
var exists = searcher.IsExists(geoName);
Write2Console('\'' + geoName + '\'' + new string(' ', Math.Max(max_len, geoName.Length) - geoName.Length) + " => HashtableSearcher.IsExists: " + exists,
(exists) ? ConsoleColor.White : ConsoleColor.Yellow);
}
catch (Exception e)
{
Write2Console('\'' + geoName + '\'' + new string(' ', Math.Max(max_len, geoName.Length) - geoName.Length) + " => " + e.ToString(), ConsoleColor.Red);
}
}
public static void IsExists2(this HashtableSearcher searcher, string geoName)
{
//try
//{
var exists = searcher.IsExists(geoName /*.HashtableEngineNormlizeText()*/);
if (!exists)
{
Write2Console('\'' + geoName + "' => searcher.IsExists: " + exists, ConsoleColor.Yellow);
}
//}
//catch (Exception e)
//{
// Write2Console( '\'' + geoName + "' => " + e.ToString(), ConsoleColor.Red );
//}
}
static void test_Search_big(string dataFileName, Encoding dataFileEncoding)
{
Console.WriteLine("Begin fetch test data for search...");
var testList = new List <string>(4000000);
using (var sr = new StreamReader(dataFileName, dataFileEncoding))
{
while (!sr.EndOfStream)
{
var text = sr.ReadLine();
if (!string.IsNullOrEmpty(text))
{
testList.Add(text);
}
}
}
Console.WriteLine("End fetch test data, all count: " + testList.Count + '.' + Environment.NewLine);
Console.WriteLine("Begin full search...");
using (var searcher = new HashtableSearcher(dataFileName, HashtableEngineHelper.NormlizeText))
{
var sw = Stopwatch.StartNew();
var i = 0;
foreach (var text in testList)
{
//searcher.IsExists2( text );
i++;
var exists = searcher.IsExists(text);
if (!exists)
{
Extensions.Write2Console('\'' + text + "' => searcher.IsExists: " + exists + ", row number: " + i, ConsoleColor.Yellow);
}
}
sw.Stop();
Console.WriteLine("End full search, elapsed: " + sw.Elapsed + ", all count: " + testList.Count + '.' + Environment.NewLine);
}
#region [.commented. disk read.]
/*Console.WriteLine("Begin full search...");
* using ( var searcher = new HashtableSearcher( dataFileName ) )
* {
* var sw = Stopwatch.StartNew();
* var i = 0;
* using ( var sr = new StreamReader( dataFileName, dataFileEncoding ) )
* {
* while ( !sr.EndOfStream )
* {
* var geo_name = sr.ReadLine();
* if ( !string.IsNullOrEmpty( geo_name ) )
* {
* searcher.IsExistsEx( geo_name );
* i++;
* }
* }
* }
* sw.Stop();
*
* Console.WriteLine( "End full search, elapsed: " + sw.Elapsed + ", all count: " + i );
* }*/
#endregion
}