public void ImportGeoIPCountryData(string path, bool ignoreInvalidRows) { _cancelImport = false; FileStream fs = null; IsRunning = true; try { var files = Directory.EnumerateFiles(path, "GeoIPCountryWhois.csv").ToList(); if (!files.Any()) { IsRunning = false; if (OnComplete != null) { OnComplete.Invoke(this, new EventArgs()); } _progressText.Invoke("could not locate file 'GeoIPCountryWhois.csv', aborting."); return; } List <IBulkCopyItem> geopIpCountries = new List <IBulkCopyItem>(); string fileName = files.First(); fs = File.OpenRead(fileName); StreamReader sr = new StreamReader(fs); _progressCallback.Invoke(0); long bytesRead = 0; while (!sr.EndOfStream) { if (_cancelImport) { return; } string lineData = sr.ReadLine(); if (lineData == null) { break; } if (lineData.Length > 0 && lineData[0] == '#') { continue; } bytesRead += lineData.Length; _progressCallback.Invoke((double)fs.Length / bytesRead); MatchCollection mc = Regex.Matches(lineData, "\".*?\""); try { if (mc.Count == 6) { var geoIpCountry = new GeoIPCountryBo() { IPAddressFrom = mc[0].Value.Replace("\"", ""), IPAddressTo = mc[1].Value.Replace("\"", ""), IPFrom = long.Parse(mc[2].Value.Replace("\"", "")), IPTo = long.Parse(mc[3].Value.Replace("\"", "")), CountryCode = mc[4].Value.Replace("\"", ""), CountryName = mc[5].Value.Replace("\"", ""), }; geopIpCountries.Add(new BulkCopyItemBo(geoIpCountry)); } } catch (Exception ex) { _progressText.Invoke("Exception while parsing GeoIPCountry row: " + ex.Message); if (!ignoreInvalidRows) { return; } } } fs.Close(); fs = null; _progressText.Invoke(string.Format("Finished parsing {0} rows from import file. Starting database import", geopIpCountries.Count)); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); _geoIpRepository = new GeoIPRepository(); // GeoIPCountryBo column definition List <ColumnDefinition> geoIpCountryDefinitions = new List <ColumnDefinition>(); geoIpCountryDefinitions.Add(new ColumnDefinition() { Index = 0, ColumnName = "IPFrom", ColumnDataType = typeof(long) }); geoIpCountryDefinitions.Add(new ColumnDefinition() { Index = 1, ColumnName = "IPTo", ColumnDataType = typeof(long) }); geoIpCountryDefinitions.Add(new ColumnDefinition() { Index = 2, ColumnName = "IPAddressFrom", ColumnDataType = typeof(string) }); geoIpCountryDefinitions.Add(new ColumnDefinition() { Index = 3, ColumnName = "IPAddressTo", ColumnDataType = typeof(string) }); geoIpCountryDefinitions.Add(new ColumnDefinition() { Index = 4, ColumnName = "CountryCode", ColumnDataType = typeof(string) }); geoIpCountryDefinitions.Add(new ColumnDefinition() { Index = 5, ColumnName = "CountryName", ColumnDataType = typeof(string) }); _geoIpRepository.ImportGeoIPCountryList(geopIpCountries, geoIpCountryDefinitions, d => _progressCallback(d)); stopwatch.Stop(); _progressText.Invoke("Database import completed after: " + stopwatch.Elapsed); if (OnComplete != null) { OnComplete.Invoke(this, new EventArgs()); } } catch (Exception ex) { _errorStringBuilder.AppendLine(ex.Message); if (OnError != null) { OnError.Invoke(this, new ErrorEventArgs(ex)); } } finally { if (fs != null) { fs.Close(); } IsRunning = false; } }
public void ImportGeoIPCityData(string path, bool ignoreInvalidRows) { _cancelImport = false; IsRunning = true; FileStream fs = null; try { Stopwatch stopwatch = new Stopwatch(); string blockImportFileName = Directory.EnumerateFiles(path, "GeoLiteCity-Blocks.csv").SingleOrDefault(); if (blockImportFileName == null) { IsRunning = false; if (OnComplete != null) { OnComplete.Invoke(this, new EventArgs()); } _progressText.Invoke("could not locate file 'GeoLiteCity-Blocks.csv', aborting."); return; } string cityImportFileName = Directory.EnumerateFiles(path, "GeoLiteCity-Location.csv").SingleOrDefault(); if (cityImportFileName == null) { IsRunning = false; if (OnComplete != null) { OnComplete.Invoke(this, new EventArgs()); } _progressText.Invoke("could not locate file 'GeoLiteCity-Location.csv', aborting."); return; } var geopIPBlocks = new ConcurrentBag <IBulkCopyItem>(); var geoIPCities = new ConcurrentBag <IBulkCopyItem>(); fs = File.OpenRead(blockImportFileName); var sr = new StreamReader(fs); _progressCallback.Invoke(0); stopwatch.Start(); bool hasInvalidRow = false; string[] lineBuffer = new string[1000]; while (!sr.EndOfStream) { if (_cancelImport) { IsRunning = false; if (OnComplete != null) { OnComplete.Invoke(this, new EventArgs()); } return; } if (stopwatch.ElapsedMilliseconds >= 100) { _progressCallback.Invoke((double)fs.Position / fs.Length); stopwatch.Restart(); } int itemsRead = 0; for (int i = 0; i < lineBuffer.Length; i++) { lineBuffer[i] = sr.ReadLine(); if (lineBuffer[i] == null) { break; } itemsRead++; } Parallel.For(0, itemsRead, i => { try { IBulkCopyItem geoBlockItem = ParseGeoIpBlock(lineBuffer[i]); if (geoBlockItem != null) { geopIPBlocks.Add(geoBlockItem); } } catch (Exception ex) { hasInvalidRow = true; _progressText.Invoke("Exception while parsing GeoIPBlock row: " + lineBuffer[i] + "\n" + ex.Message); } }); if (!ignoreInvalidRows && hasInvalidRow) { return; } } fs.Close(); fs = null; _progressText.Invoke(string.Format("Finished parsing {0} rows from GeoBlockIp import file.", geopIPBlocks.Count)); fs = File.OpenRead(cityImportFileName); sr = new StreamReader(fs); _progressCallback.Invoke(0); stopwatch.Restart(); while (!sr.EndOfStream) { if (_cancelImport) { IsRunning = false; if (OnComplete != null) { OnComplete.Invoke(this, new EventArgs()); } return; } if (stopwatch.ElapsedMilliseconds >= 100) { _progressCallback.Invoke((double)fs.Position / fs.Length); stopwatch.Restart(); } int itemsRead = 0; for (int i = 0; i < lineBuffer.Length; i++) { lineBuffer[i] = sr.ReadLine(); if (lineBuffer[i] == null) { break; } itemsRead++; } Parallel.For(0, itemsRead, i => { try { IBulkCopyItem geoIpCityItem = ParseGeoIpCity(lineBuffer[i]); if (geoIpCityItem != null) { geoIPCities.Add(geoIpCityItem); } } catch (Exception ex) { hasInvalidRow = true; _progressText.Invoke("Exception while parsing GeoIpCity row: " + lineBuffer[i] + "\n" + ex.Message); } }); if (!ignoreInvalidRows && hasInvalidRow) { return; } } fs.Close(); fs = null; stopwatch.Stop(); _progressText.Invoke(string.Format("Finished parsing {0} rows from GeoIPCity import file. Starting database import", geoIPCities.Count)); stopwatch.Restart(); _geoIpRepository = new GeoIPRepository(); _geoIpRepository.ImportGeoIPCitiesList(geopIPBlocks.ToList(), geoIPCities.ToList(), getGeoIpBlockColumnDefinitions(), getGeoIpCityColumnDefinitions(), d => _progressCallback(d)); stopwatch.Stop(); _progressText.Invoke("Database import completed after: " + stopwatch.Elapsed); } catch (Exception ex) { if (OnError != null) { OnError.Invoke(this, new ErrorEventArgs(ex)); } } finally { if (fs != null) { fs.Close(); } IsRunning = false; } if (OnComplete != null) { OnComplete.Invoke(this, new EventArgs()); } }