public GeobaseEngineCombined(string fileName) { var stopwatchAll = Stopwatch.StartNew(); // init sizes HeaderLengh = Extentions.GetBytesLength(typeof(GeobaseHeaderDirrect)); RangesLengh = Extentions.GetBytesLength(typeof(GeobaseIpRangeDirrect)); LocationLengh = Extentions.GetBytesLength(typeof(GeobaseLocationDirrect)); CityIndexLengh = Extentions.GetBytesLength(typeof(GeobaseCityIndexDirrect)); CityOffsetInLocation = Extentions.GetBytesLength(typeof(GeobaseLocationDirrect), "Country") + Extentions.GetBytesLength(typeof(GeobaseLocationDirrect), "Region") + Extentions.GetBytesLength(typeof(GeobaseLocationDirrect), "Postal"); LocationCityBytesLenght = Extentions.GetBytesLength(typeof(GeobaseLocationDirrect), "City"); // init data Bytes = File.ReadAllBytes(fileName); Header = ByteHelper.BytesToStruct <GeobaseHeaderMarshal>(ByteHelper.GetBytes(Bytes, 0, HeaderLengh)); var cityIndexesOffsets = ByteHelper.BytesToStruct <GeobaseLocationCityIndexesCombined>(ByteHelper.GetBytes(Bytes, (int)CitiesFromTo.Item1, (int)(CitiesFromTo.Item2 - CitiesFromTo.Item1))); _cityIndexPrepared = new int[Records]; for (int i = 0; i < Records; i++) { _cityIndexPrepared[i] = (int)(cityIndexesOffsets.CityIndexes[i].LocationOffset / LocationLengh); } stopwatchAll.Stop(); Debug.WriteLine("GeobaseEngineCombined, DB loading time: " + stopwatchAll.Elapsed.TotalMilliseconds + " ms"); }
public GeobaseEngineMarshal(string fileName) { var stopwatchAll = Stopwatch.StartNew(); HeaderLengh = Extentions.GetBytesLength(typeof(GeobaseHeaderMarshal)); RangesLengh = Extentions.GetBytesLength(typeof(GeobaseIpRangeMarshal)); LocationLengh = Extentions.GetBytesLength(typeof(GeobaseLocationMarshal)); CityIndexLengh = Extentions.GetBytesLength(typeof(GeobaseCityIndexMarshal)); Bytes = File.ReadAllBytes(fileName); //_data = ByteHelper.BytesToStruct<GeobaseDataMarshal>(Bytes); // old approach (slow, but simple) #region parrallel loading _data = new GeobaseDataMarshal(); _data.Header = ByteHelper.BytesToStruct <GeobaseHeaderMarshal>(ByteHelper.GetBytes(Bytes, 0, HeaderLengh)); var locationPartsCount = 50; var locationParts = new ConcurrentBag <GeobaseLocationsPart>(); var locationTasks = new Task[locationPartsCount + 2]; if (Records % locationPartsCount != 0) { throw new Exception("Wrong locationPartsCount"); } var locationPartSize = Records / locationPartsCount; var locationPartSizeBytes = LocationLengh * locationPartSize; for (var i = 0; i < locationPartsCount; i++) { var i1 = i; locationTasks[i] = Task.Factory.StartNew( () => { locationParts.Add(new GeobaseLocationsPart { Locations = ByteHelper.BytesToStruct <GeobaseLocationsMarshal>( ByteHelper.GetBytes(Bytes , (int)_data.Header.OffsetLocations + locationPartSizeBytes * i1 , locationPartSizeBytes)).Locations , PartNumber = i1 }); }) ; } locationTasks[locationPartsCount] = Task.Factory.StartNew( () => { _data.IpRanges = ByteHelper.BytesToStruct <GeobaseIpRangesMarshal>(ByteHelper.GetBytes(Bytes, (int)_data.Header.OffsetRanges, Records * RangesLengh)).IpRanges; }); locationTasks[locationPartsCount + 1] = Task.Factory.StartNew( () => { _data.CityIndexes = ByteHelper.BytesToStruct <GeobaseCityIndexesMarshal>(ByteHelper.GetBytes(Bytes, (int)_data.Header.OffsetCities, Records * CityIndexLengh)).CityIndexes; }); Task.WaitAll(locationTasks); _data.Locations = new GeobaseLocationMarshal[Records]; var j = 0; foreach (var locationPart in locationParts.OrderBy(x => x.PartNumber)) { Array.Copy(locationPart.Locations, 0, _data.Locations, j * locationPartSize, locationPartSize); j++; } #endregion _cityIndexPrepared = new int[Records]; for (int i = 0; i < Records; i++) { _cityIndexPrepared[i] = (int)(_data.CityIndexes[i].LocationOffset / LocationLengh); } stopwatchAll.Stop(); Debug.WriteLine("GeobaseEngineMarshal, DB loading time: " + stopwatchAll.Elapsed.TotalMilliseconds + " ms"); }
private GeobaseLocationView GetLocation(byte[] bytes, int offset) { return(TransformationHelper.GetLocationView( ByteHelper.BytesToStruct <GeobaseLocationMarshal>(ByteHelper.GetBytes(bytes, offset, LocationLengh)) )); }
public IEnumerable <GeobaseLocationView> FindLocationsByCity(string city) { var stopwatch = Stopwatch.StartNew(); // binary search int min = 0; int max = _cityIndexPrepared.Length - 1; bool found = false; while (min <= max) { int mid = (min + max) / 2; var locationOffset = LocationsFromTo.Item1 + _cityIndexPrepared[mid] * LocationLengh; var locationCityOffset = (locationOffset + CityOffsetInLocation); var locationCity = ByteHelper.BytesToStruct <GeobaseLocationCityCombined>( ByteHelper.GetBytes(Bytes, locationCityOffset, LocationCityBytesLenght)) .City; var comparisonResult = string.CompareOrdinal(city, locationCity); if (comparisonResult == 0) { yield return(GetLocation(Bytes, locationOffset)); var previousCity = city; var pointer = mid - 1; while (pointer >= 0) { locationOffset = LocationsFromTo.Item1 + _cityIndexPrepared[pointer] * LocationLengh; locationCityOffset = (locationOffset + CityOffsetInLocation); previousCity = ByteHelper.BytesToStruct <GeobaseLocationCityCombined>( ByteHelper.GetBytes(Bytes, locationCityOffset, LocationCityBytesLenght)) .City; if (string.CompareOrdinal(city, previousCity) == 0) { yield return(GetLocation(Bytes, locationOffset)); } else { break; } pointer--; } var nextCity = city; pointer = mid + 1; while (pointer < _cityIndexPrepared.Length) { locationOffset = LocationsFromTo.Item1 + _cityIndexPrepared[pointer] * LocationLengh; locationCityOffset = (locationOffset + CityOffsetInLocation); nextCity = ByteHelper.BytesToStruct <GeobaseLocationCityCombined>( ByteHelper.GetBytes(Bytes, locationCityOffset, LocationCityBytesLenght)) .City; if (string.CompareOrdinal(city, nextCity) == 0) { yield return(GetLocation(Bytes, locationOffset)); } else { break; } pointer++; } found = true; break; } else if (comparisonResult < 0) { max = mid - 1; } else { min = mid + 1; } } if (!found) { Debug.WriteLine("GeobaseEngineCombined, FindLocationsByCity, city = " + city + ", time: " + stopwatch.Elapsed.TotalMilliseconds + " ms. Result: NOT FOUND"); } Debug.WriteLine("GeobaseEngineCombined, FindLocationsByCity, city = " + city + ", time: " + stopwatch.Elapsed.TotalMilliseconds + " ms"); }
public void PerformanceTest() { var count = 50000; var stopwatch = Stopwatch.StartNew(); var fileName = Path.Combine(Directory.GetCurrentDirectory(), "Data\\geobase.dat"); var engineDirrect = new GeobaseEngineDirrect(fileName); var offset = (int)engineDirrect.Header.OffsetLocations; var resultsMarshal = new List <GeobaseLocationView>(); for (int i = 0; i < count; i++) { var toAdd = ByteHelper.BytesToStruct <GeobaseLocationMarshal>( new ArraySegment <byte>( engineDirrect.Bytes, offset + i * engineDirrect.LocationLengh, engineDirrect.LocationLengh).ToArray() ); resultsMarshal.Add(TransformationHelper.GetLocationView(toAdd)); } stopwatch.Stop(); Console.WriteLine("BytesToObjectTest.PerformanceTest - get locations from bytes, marshal access time: " + stopwatch.Elapsed.TotalMilliseconds + " ms."); stopwatch.Restart(); var resultsDirrect = new List <GeobaseLocationView>(); for (int i = 0; i < count; i++) { resultsDirrect.Add(ByteHelper.GetLocation(engineDirrect.Bytes, offset + i * engineDirrect.LocationLengh)); } stopwatch.Stop(); Console.WriteLine("BytesToObjectTest.PerformanceTest - get locations from bytes, dirrect access time: " + stopwatch.Elapsed.TotalMilliseconds + " ms."); stopwatch.Restart(); Assert.AreEqual(resultsDirrect.Count, count); Assert.AreEqual(resultsMarshal.Count, count); Assert.AreEqual(resultsDirrect[0].city, resultsMarshal[0].city); Assert.AreEqual(resultsDirrect[0].country, resultsMarshal[0].country); Assert.AreEqual(resultsDirrect[0].latitude, resultsMarshal[0].latitude); Assert.AreEqual(resultsDirrect[0].longitude, resultsMarshal[0].longitude); Assert.AreEqual(resultsDirrect[0].organization, resultsMarshal[0].organization); Assert.AreEqual(resultsDirrect[0].postal, resultsMarshal[0].postal); Assert.AreEqual(resultsDirrect[0].region, resultsMarshal[0].region); var index = count - 10; Assert.AreEqual(resultsDirrect[index].city, resultsMarshal[index].city); Assert.AreEqual(resultsDirrect[index].country, resultsMarshal[index].country); Assert.AreEqual(resultsDirrect[index].latitude, resultsMarshal[index].latitude); Assert.AreEqual(resultsDirrect[index].longitude, resultsMarshal[index].longitude); Assert.AreEqual(resultsDirrect[index].organization, resultsMarshal[index].organization); Assert.AreEqual(resultsDirrect[index].postal, resultsMarshal[index].postal); Assert.AreEqual(resultsDirrect[index].region, resultsMarshal[index].region); index = count - 1; Assert.AreEqual(resultsDirrect[index].city, resultsMarshal[index].city); Assert.AreEqual(resultsDirrect[index].country, resultsMarshal[index].country); Assert.AreEqual(resultsDirrect[index].latitude, resultsMarshal[index].latitude); Assert.AreEqual(resultsDirrect[index].longitude, resultsMarshal[index].longitude); Assert.AreEqual(resultsDirrect[index].organization, resultsMarshal[index].organization); Assert.AreEqual(resultsDirrect[index].postal, resultsMarshal[index].postal); Assert.AreEqual(resultsDirrect[index].region, resultsMarshal[index].region); }