public GeobaseEngineCombined(string fileName)
{
var stopwatchAll = Stopwatch.StartNew();
// init sizes
HeaderLengh = Extentions.GetBytesLength(typeof(GeobaseHeaderDirrect));
RangesLengh = Extentions.GetBytesLength(typeof(GeobaseIpRangeDirrect));
LocationLengh = Extentions.GetBytesLength(typeof(GeobaseLocationDirrect));
CityIndexLengh = Extentions.GetBytesLength(typeof(GeobaseCityIndexDirrect));
CityOffsetInLocation = Extentions.GetBytesLength(typeof(GeobaseLocationDirrect), "Country")
+ Extentions.GetBytesLength(typeof(GeobaseLocationDirrect), "Region")
+ Extentions.GetBytesLength(typeof(GeobaseLocationDirrect), "Postal");
LocationCityBytesLenght = Extentions.GetBytesLength(typeof(GeobaseLocationDirrect), "City");
// init data
Bytes = File.ReadAllBytes(fileName);
Header = ByteHelper.BytesToStruct <GeobaseHeaderMarshal>(ByteHelper.GetBytes(Bytes, 0, HeaderLengh));
var cityIndexesOffsets = ByteHelper.BytesToStruct <GeobaseLocationCityIndexesCombined>(ByteHelper.GetBytes(Bytes, (int)CitiesFromTo.Item1, (int)(CitiesFromTo.Item2 - CitiesFromTo.Item1)));
_cityIndexPrepared = new int[Records];
for (int i = 0; i < Records; i++)
{
_cityIndexPrepared[i] = (int)(cityIndexesOffsets.CityIndexes[i].LocationOffset / LocationLengh);
}
stopwatchAll.Stop();
Debug.WriteLine("GeobaseEngineCombined, DB loading time: " + stopwatchAll.Elapsed.TotalMilliseconds + " ms");
}
public GeobaseEngineMarshal(string fileName)
{
var stopwatchAll = Stopwatch.StartNew();
HeaderLengh = Extentions.GetBytesLength(typeof(GeobaseHeaderMarshal));
RangesLengh = Extentions.GetBytesLength(typeof(GeobaseIpRangeMarshal));
LocationLengh = Extentions.GetBytesLength(typeof(GeobaseLocationMarshal));
CityIndexLengh = Extentions.GetBytesLength(typeof(GeobaseCityIndexMarshal));
Bytes = File.ReadAllBytes(fileName);
//_data = ByteHelper.BytesToStruct<GeobaseDataMarshal>(Bytes); // old approach (slow, but simple)
#region parrallel loading
_data = new GeobaseDataMarshal();
_data.Header = ByteHelper.BytesToStruct <GeobaseHeaderMarshal>(ByteHelper.GetBytes(Bytes, 0, HeaderLengh));
var locationPartsCount = 50;
var locationParts = new ConcurrentBag <GeobaseLocationsPart>();
var locationTasks = new Task[locationPartsCount + 2];
if (Records % locationPartsCount != 0)
{
throw new Exception("Wrong locationPartsCount");
}
var locationPartSize = Records / locationPartsCount;
var locationPartSizeBytes = LocationLengh * locationPartSize;
for (var i = 0; i < locationPartsCount; i++)
{
var i1 = i;
locationTasks[i] =
Task.Factory.StartNew(
() =>
{
locationParts.Add(new GeobaseLocationsPart
{
Locations = ByteHelper.BytesToStruct <GeobaseLocationsMarshal>(
ByteHelper.GetBytes(Bytes
, (int)_data.Header.OffsetLocations + locationPartSizeBytes * i1
, locationPartSizeBytes)).Locations
,
PartNumber = i1
});
})
;
}
locationTasks[locationPartsCount] = Task.Factory.StartNew(
() => { _data.IpRanges = ByteHelper.BytesToStruct <GeobaseIpRangesMarshal>(ByteHelper.GetBytes(Bytes, (int)_data.Header.OffsetRanges, Records * RangesLengh)).IpRanges; });
locationTasks[locationPartsCount + 1] = Task.Factory.StartNew(
() => { _data.CityIndexes = ByteHelper.BytesToStruct <GeobaseCityIndexesMarshal>(ByteHelper.GetBytes(Bytes, (int)_data.Header.OffsetCities, Records * CityIndexLengh)).CityIndexes; });
Task.WaitAll(locationTasks);
_data.Locations = new GeobaseLocationMarshal[Records];
var j = 0;
foreach (var locationPart in locationParts.OrderBy(x => x.PartNumber))
{
Array.Copy(locationPart.Locations, 0, _data.Locations, j * locationPartSize, locationPartSize);
j++;
}
#endregion
_cityIndexPrepared = new int[Records];
for (int i = 0; i < Records; i++)
{
_cityIndexPrepared[i] = (int)(_data.CityIndexes[i].LocationOffset / LocationLengh);
}
stopwatchAll.Stop();
Debug.WriteLine("GeobaseEngineMarshal, DB loading time: " + stopwatchAll.Elapsed.TotalMilliseconds + " ms");
}
private GeobaseLocationView GetLocation(byte[] bytes, int offset)
{
return(TransformationHelper.GetLocationView(
ByteHelper.BytesToStruct <GeobaseLocationMarshal>(ByteHelper.GetBytes(bytes, offset, LocationLengh))
));
}
public IEnumerable <GeobaseLocationView> FindLocationsByCity(string city)
{
var stopwatch = Stopwatch.StartNew();
// binary search
int min = 0;
int max = _cityIndexPrepared.Length - 1;
bool found = false;
while (min <= max)
{
int mid = (min + max) / 2;
var locationOffset = LocationsFromTo.Item1 + _cityIndexPrepared[mid] * LocationLengh;
var locationCityOffset = (locationOffset + CityOffsetInLocation);
var locationCity = ByteHelper.BytesToStruct <GeobaseLocationCityCombined>(
ByteHelper.GetBytes(Bytes, locationCityOffset, LocationCityBytesLenght))
.City;
var comparisonResult = string.CompareOrdinal(city, locationCity);
if (comparisonResult == 0)
{
yield return(GetLocation(Bytes, locationOffset));
var previousCity = city;
var pointer = mid - 1;
while (pointer >= 0)
{
locationOffset = LocationsFromTo.Item1 + _cityIndexPrepared[pointer] * LocationLengh;
locationCityOffset = (locationOffset + CityOffsetInLocation);
previousCity = ByteHelper.BytesToStruct <GeobaseLocationCityCombined>(
ByteHelper.GetBytes(Bytes, locationCityOffset, LocationCityBytesLenght))
.City;
if (string.CompareOrdinal(city, previousCity) == 0)
{
yield return(GetLocation(Bytes, locationOffset));
}
else
{
break;
}
pointer--;
}
var nextCity = city;
pointer = mid + 1;
while (pointer < _cityIndexPrepared.Length)
{
locationOffset = LocationsFromTo.Item1 + _cityIndexPrepared[pointer] * LocationLengh;
locationCityOffset = (locationOffset + CityOffsetInLocation);
nextCity = ByteHelper.BytesToStruct <GeobaseLocationCityCombined>(
ByteHelper.GetBytes(Bytes, locationCityOffset, LocationCityBytesLenght))
.City;
if (string.CompareOrdinal(city, nextCity) == 0)
{
yield return(GetLocation(Bytes, locationOffset));
}
else
{
break;
}
pointer++;
}
found = true;
break;
}
else if (comparisonResult < 0)
{
max = mid - 1;
}
else
{
min = mid + 1;
}
}
if (!found)
{
Debug.WriteLine("GeobaseEngineCombined, FindLocationsByCity, city = " + city + ", time: " + stopwatch.Elapsed.TotalMilliseconds + " ms. Result: NOT FOUND");
}
Debug.WriteLine("GeobaseEngineCombined, FindLocationsByCity, city = " + city + ", time: " + stopwatch.Elapsed.TotalMilliseconds + " ms");
}
public void PerformanceTest()
{
var count = 50000;
var stopwatch = Stopwatch.StartNew();
var fileName = Path.Combine(Directory.GetCurrentDirectory(), "Data\\geobase.dat");
var engineDirrect = new GeobaseEngineDirrect(fileName);
var offset = (int)engineDirrect.Header.OffsetLocations;
var resultsMarshal = new List <GeobaseLocationView>();
for (int i = 0; i < count; i++)
{
var toAdd = ByteHelper.BytesToStruct <GeobaseLocationMarshal>(
new ArraySegment <byte>(
engineDirrect.Bytes, offset + i * engineDirrect.LocationLengh, engineDirrect.LocationLengh).ToArray()
);
resultsMarshal.Add(TransformationHelper.GetLocationView(toAdd));
}
stopwatch.Stop();
Console.WriteLine("BytesToObjectTest.PerformanceTest - get locations from bytes, marshal access time: " + stopwatch.Elapsed.TotalMilliseconds + " ms.");
stopwatch.Restart();
var resultsDirrect = new List <GeobaseLocationView>();
for (int i = 0; i < count; i++)
{
resultsDirrect.Add(ByteHelper.GetLocation(engineDirrect.Bytes, offset + i * engineDirrect.LocationLengh));
}
stopwatch.Stop();
Console.WriteLine("BytesToObjectTest.PerformanceTest - get locations from bytes, dirrect access time: " + stopwatch.Elapsed.TotalMilliseconds + " ms.");
stopwatch.Restart();
Assert.AreEqual(resultsDirrect.Count, count);
Assert.AreEqual(resultsMarshal.Count, count);
Assert.AreEqual(resultsDirrect[0].city, resultsMarshal[0].city);
Assert.AreEqual(resultsDirrect[0].country, resultsMarshal[0].country);
Assert.AreEqual(resultsDirrect[0].latitude, resultsMarshal[0].latitude);
Assert.AreEqual(resultsDirrect[0].longitude, resultsMarshal[0].longitude);
Assert.AreEqual(resultsDirrect[0].organization, resultsMarshal[0].organization);
Assert.AreEqual(resultsDirrect[0].postal, resultsMarshal[0].postal);
Assert.AreEqual(resultsDirrect[0].region, resultsMarshal[0].region);
var index = count - 10;
Assert.AreEqual(resultsDirrect[index].city, resultsMarshal[index].city);
Assert.AreEqual(resultsDirrect[index].country, resultsMarshal[index].country);
Assert.AreEqual(resultsDirrect[index].latitude, resultsMarshal[index].latitude);
Assert.AreEqual(resultsDirrect[index].longitude, resultsMarshal[index].longitude);
Assert.AreEqual(resultsDirrect[index].organization, resultsMarshal[index].organization);
Assert.AreEqual(resultsDirrect[index].postal, resultsMarshal[index].postal);
Assert.AreEqual(resultsDirrect[index].region, resultsMarshal[index].region);
index = count - 1;
Assert.AreEqual(resultsDirrect[index].city, resultsMarshal[index].city);
Assert.AreEqual(resultsDirrect[index].country, resultsMarshal[index].country);
Assert.AreEqual(resultsDirrect[index].latitude, resultsMarshal[index].latitude);
Assert.AreEqual(resultsDirrect[index].longitude, resultsMarshal[index].longitude);
Assert.AreEqual(resultsDirrect[index].organization, resultsMarshal[index].organization);
Assert.AreEqual(resultsDirrect[index].postal, resultsMarshal[index].postal);
Assert.AreEqual(resultsDirrect[index].region, resultsMarshal[index].region);
}